Source code for backend.mockremote.builder

import os
import pipes
import socket
from subprocess import Popen
import time
from urlparse import urlparse

from ansible.runner import Runner
from backend.vm_manage import PUBSUB_INTERRUPT_BUILDER
from ..helpers import get_redis_connection

from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError, VmError

from ..constants import mockchain, rsync, DEF_BUILD_TIMEOUT


[docs]class Builder(object): def __init__(self, opts, hostname, job, logger): self.opts = opts self.hostname = hostname self.job = job self.timeout = self.job.timeout or self.opts.timeout self.repos = [] self.log = logger self.buildroot_pkgs = self.job.buildroot_pkgs or "" self._remote_tempdir = self.opts.remote_tempdir self._remote_basedir = self.opts.remote_basedir self.remote_pkg_path = None self.remote_pkg_name = None # if we're at this point we've connected and done stuff on the host self.conn = self._create_ans_conn() self.root_conn = self._create_ans_conn(username="root") @property
[docs] def remote_build_dir(self): return self.tempdir + "/build/"
@property def tempdir(self): if self._remote_tempdir: return self._remote_tempdir create_tmpdir_cmd = "/bin/mktemp -d {0}/{1}-XXXXX".format( self._remote_basedir, "mockremote") results = self._run_ansible(create_tmpdir_cmd) tempdir = None # TODO: use check_for_ans_error for _, resdict in results["contacted"].items(): tempdir = resdict["stdout"] # if still nothing then we"ve broken if not tempdir: raise BuilderError("Could not make tmpdir on {0}".format( self.hostname)) self._run_ansible("/bin/chmod 755 {0}".format(tempdir)) self._remote_tempdir = tempdir return self._remote_tempdir @tempdir.setter
[docs] def tempdir(self, value): self._remote_tempdir = value
[docs] def _create_ans_conn(self, username=None): ans_conn = Runner(remote_user=username or self.opts.build_user, host_list=self.hostname + ",", pattern=self.hostname, forks=1, transport=self.opts.ssh.transport, timeout=self.timeout) return ans_conn
[docs] def run_ansible_with_check(self, cmd, module_name=None, as_root=False, err_codes=None, success_codes=None): results = self._run_ansible(cmd, module_name, as_root) try: check_for_ans_error( results, self.hostname, err_codes, success_codes) except AnsibleResponseError as response_error: raise AnsibleCallError( msg="Failed to execute ansible command", cmd=cmd, module_name=module_name, as_root=as_root, return_code=response_error.return_code, stdout=response_error.stdout, stderr=response_error.stderr ) return results
[docs] def _run_ansible(self, cmd, module_name=None, as_root=False): """ Executes single ansible module :param str cmd: module command :param str module_name: name of the invoked module :param bool as_root: :return: ansible command result """ if as_root: conn = self.root_conn else: conn = self.conn conn.module_name = module_name or "shell" conn.module_args = str(cmd) return conn.run()
[docs] def _get_remote_results_dir(self): if any(x is None for x in [self.remote_build_dir, self.remote_pkg_name, self.job.chroot]): return None # the pkg will build into a dir by mockchain named: # $tempdir/build/results/$chroot/$packagename return os.path.normpath(os.path.join( self.remote_build_dir, "results", self.job.chroot, self.remote_pkg_name))
[docs] def modify_mock_chroot_config(self): """ Modify mock config for current chroot. Packages in buildroot_pkgs are added to minimal buildroot """ if ("'{0} '".format(self.buildroot_pkgs) != pipes.quote(str(self.buildroot_pkgs) + ' ')): # just different test if it contains only alphanumeric characters # allowed in packages name raise BuilderError("Do not try this kind of attack on me") self.log.info("putting {0} into minimal buildroot of {1}" .format(self.buildroot_pkgs, self.job.chroot)) kwargs = { "chroot": self.job.chroot, "pkgs": self.buildroot_pkgs, "net_enabled": "True" if self.job.enable_net else "False", } buildroot_cmd = ( "dest=/etc/mock/{chroot}.cfg" " line=\"config_opts['chroot_setup_cmd'] = 'install \\1 {pkgs}'\"" " regexp=\"^.*chroot_setup_cmd.*(@buildsys-build|buildsys-build buildsys-macros).*$\"" " backrefs=yes" ) buildroot_custom_cmd = ( "dest=/etc/mock/{chroot}.cfg" " line=\"config_opts['chroot_setup_cmd'] = 'install {pkgs}'\"" " regexp=\"config_opts['chroot_setup_cmd'] = ''$\"" " backrefs=yes" ) set_networking_cmd = ( "dest=/etc/mock/{chroot}.cfg" " line=\"config_opts['use_host_resolv'] = {net_enabled}\"" " regexp=\"^.*use_host_resolv.*$\"" ) try: self.run_ansible_with_check(set_networking_cmd.format(**kwargs), module_name="lineinfile", as_root=True) if self.buildroot_pkgs: self.run_ansible_with_check(buildroot_cmd.format(**kwargs), module_name="lineinfile", as_root=True) self.run_ansible_with_check(buildroot_custom_cmd.format(**kwargs), module_name="lineinfile", as_root=True) except BuilderError as err: self.log.exception(err) raise
[docs] def collect_built_packages(self): self.log.info("Listing built binary packages") results = self._run_ansible( "cd {0} && " "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " "done".format(pipes.quote(self._get_remote_results_dir())) ) built_packages = list(results["contacted"].values())[0][u"stdout"] self.log.info("Built packages:\n{}".format(built_packages)) return built_packages
[docs] def check_build_success(self): successfile = os.path.join(self._get_remote_results_dir(), "success") ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile)) check_for_ans_error(ansible_test_results, self.hostname)
[docs] def download_job_pkg_to_builder(self): repo_url = "{}/{}.git".format(self.opts.dist_git_url, self.job.git_repo) self.log.info("Cloning Dist Git repo {}, branch {}, hash {}".format( self.job.git_repo, self.job.git_branch, self.job.git_hash)) results = self._run_ansible( "rm -rf /tmp/build_package_repo && " "mkdir /tmp/build_package_repo && " "cd /tmp/build_package_repo && " "git clone {repo_url} && " "cd {pkg_name} && " "git checkout {git_hash} && " "fedpkg-copr --dist {branch} srpm" .format(repo_url=repo_url, pkg_name=self.job.package_name, git_hash=self.job.git_hash, branch=self.job.git_branch)) # expected output: # ... # Wrote: /tmp/.../copr-ping/copr-ping-1-1.fc21.src.rpm try: self.remote_pkg_path = list(results["contacted"].values())[0][u"stdout"].split("Wrote: ")[1] self.remote_pkg_name = os.path.basename(self.remote_pkg_path).replace(".src.rpm", "") except Exception: self.log.exception("Failed to obtain srpm from dist-git") raise BuilderError("Failed to obtain srpm from dist-git: ansible results {}".format(results)) self.log.info("Got srpm to build: {}".format(self.remote_pkg_path))
[docs] def pre_process_repo_url(self, repo_url): """ Expands variables and sanitize repo url to be used for mock config """ try: parsed_url = urlparse(repo_url) if parsed_url.scheme == "copr": user = parsed_url.netloc prj = parsed_url.path.split("/")[1] repo_url = "/".join([self.opts.results_baseurl, user, prj, self.job.chroot]) else: if "rawhide" in self.job.chroot: repo_url = repo_url.replace("$releasever", "rawhide") # custom expand variables repo_url = repo_url.replace("$chroot", self.job.chroot) repo_url = repo_url.replace("$distname", self.job.chroot.split("-")[0]) return pipes.quote(repo_url) except Exception as err: self.log.exception("Failed to pre-process repo url: {}".format(err)) return None
[docs] def gen_mockchain_command(self): buildcmd = "{} -r {} -l {} ".format( mockchain, pipes.quote(self.job.chroot), pipes.quote(self.remote_build_dir)) for repo in self.job.chroot_repos_extended: repo = self.pre_process_repo_url(repo) if repo is not None: buildcmd += "-a {0} ".format(repo) for k, v in self.job.mockchain_macros.items(): mock_opt = "--define={} {}".format(k, v) buildcmd += "-m {} ".format(pipes.quote(mock_opt)) buildcmd += self.remote_pkg_path return buildcmd
[docs] def run_build_and_wait(self, buildcmd): self.log.info("executing: {0}".format(buildcmd)) self.conn.module_name = "shell" self.conn.module_args = buildcmd _, poller = self.conn.run_async(self.timeout) waited = 0 results = None # self.setup_pubsub_handler() while True: # TODO rework Builder and extrace check_pubsub, add method to interrupt build process from dispatcher # self.check_pubsub() results = poller.poll() if results["contacted"] or results["dark"]: break if waited >= self.timeout: raise BuilderTimeOutError("Build timeout expired. Time limit: {}s, time spent: {}s" .format(self.timeout, waited)) time.sleep(10) waited += 10 return results
[docs] def setup_pubsub_handler(self): self.rc = get_redis_connection(self.opts) self.ps = self.rc.pubsub(ignore_subscribe_messages=True) channel_name = PUBSUB_INTERRUPT_BUILDER.format(self.hostname) self.ps.subscribe(channel_name) self.log.info("Subscribed to vm interruptions channel {}".format(channel_name))
[docs] def check_pubsub(self): # self.log.info("Checking pubsub channel") msg = self.ps.get_message() if msg is not None and msg.get("type") == "message": raise VmError("Build interrupted by msg: {}".format(msg["data"])) # def start_build(self, pkg): # # build the pkg passed in # # add pkg to various lists # # check for success/failure of build # # # build_details = {} # self.modify_mock_chroot_config() # # # check if pkg is local or http # dest = self.check_if_pkg_local_or_http(pkg) # # # srpm version # self.update_job_pkg_version(pkg) # # # construct the mockchain command # buildcmd = self.gen_mockchain_command(dest) #
[docs] def build(self): self.modify_mock_chroot_config() # download the package to the builder self.download_job_pkg_to_builder() # construct the mockchain command buildcmd = self.gen_mockchain_command() # run the mockchain command async ansible_build_results = self.run_build_and_wait(buildcmd) # now raises BuildTimeoutError check_for_ans_error(ansible_build_results, self.hostname) # on error raises AnsibleResponseError # we know the command ended successfully but not if the pkg built # successfully self.check_build_success() return get_ans_results(ansible_build_results, self.hostname).get("stdout", "")
[docs] def download(self, target_dir): if self._get_remote_results_dir(): self.log.info("Start retrieve results for: {0}".format(self.job)) # download the pkg to destdir using rsync + ssh # # make spaces work w/our rsync command below :( destdir = "'" + target_dir.replace("'", "'\\''") + "'" # build rsync command line from the above remote_src = "{}@{}:{}/*".format(self.opts.build_user, self.hostname, self._get_remote_results_dir()) ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'" rsync_log_filepath = os.path.join(destdir, self.job.rsync_log_name) command = "{} -rlptDvH -e {} {} {}/ &> {}".format( rsync, ssh_opts, remote_src, destdir, rsync_log_filepath) # dirty magic with Popen due to IO buffering # see http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/ # alternative: use tempfile.Tempfile as Popen stdout/stderr try: cmd = Popen(command, shell=True) cmd.wait() self.log.info("End retrieve results for: {0}".format(self.job)) except Exception as error: err_msg = "Failed to download results from builder due to rsync error, see the rsync log file for details. Original error: {}".format(error) self.log.error(err_msg) raise BuilderError(err_msg) if cmd.returncode != 0: err_msg = "Failed to download results from builder due to rsync error, see the rsync log file for details." self.log.error(err_msg) raise BuilderError(err_msg, return_code=cmd.returncode)
[docs] def check(self): # do check of host try: # requires name resolve facility socket.gethostbyname(self.hostname) except IOError: raise BuilderError("{0} could not be resolved".format(self.hostname)) try: # check_for_ans_error(res, self.hostname) self.run_ansible_with_check("/bin/rpm -q mock rsync") except AnsibleCallError: raise BuilderError(msg="Build host `{0}` does not have mock or rsync installed" .format(self.hostname)) # test for path existence for mockchain and chroot config for this chroot try: self.run_ansible_with_check("/usr/bin/test -f {0}".format(mockchain)) except AnsibleCallError: raise BuilderError(msg="Build host `{}` missing mockchain binary `{}`" .format(self.hostname, mockchain)) try: self.run_ansible_with_check("/usr/bin/test -f /etc/mock/{}.cfg" .format(self.job.chroot)) except AnsibleCallError: raise BuilderError(msg="Build host `{}` missing mock config for chroot `{}`" .format(self.hostname, self.job.chroot))
[docs]def get_ans_results(results, hostname): if hostname in results["dark"]: return results["dark"][hostname] if hostname in results["contacted"]: return results["contacted"][hostname] return {}
[docs]def check_for_ans_error(results, hostname, err_codes=None, success_codes=None): """ dict includes 'msg' may include 'rc', 'stderr', 'stdout' and any other requested result codes :raises AnsibleResponseError: :raises VmError: """ if err_codes is None: err_codes = [] if success_codes is None: success_codes = [0] if ("dark" in results and hostname in results["dark"]) or \ "contacted" not in results or hostname not in results["contacted"]: raise VmError(msg="Error: Could not contact/connect to {}. raw results: {}".format(hostname, results)) error = False err_results = {} if err_codes or success_codes: if hostname in results["contacted"]: if "rc" in results["contacted"][hostname]: rc = int(results["contacted"][hostname]["rc"]) err_results["return_code"] = rc # check for err codes first if rc in err_codes: error = True err_results["msg"] = "rc {0} matched err_codes".format(rc) elif rc not in success_codes: error = True err_results["msg"] = "rc {0} not in success_codes".format(rc) elif ("failed" in results["contacted"][hostname] and results["contacted"][hostname]["failed"]): error = True err_results["msg"] = "results included failed as true" if error: for item in ["stdout", "stderr"]: if item in results["contacted"][hostname]: err_results[item] = results["contacted"][hostname][item] if error: raise AnsibleResponseError(**err_results)