# by skvidal
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
# copyright 2012 Red Hat, Inc.
import fcntl
import logging
import os
import subprocess
from munch import Munch
import time
from ..constants import DEF_BUILD_TIMEOUT, DEF_REPOS, \
DEF_BUILD_USER, DEF_MACROS
from ..exceptions import MockRemoteError, BuilderError, CreateRepoError
from ..helpers import uses_devel_repo
# TODO: replace sign & createrepo with dependency injection
from ..sign import sign_rpms_in_dir, get_pubkey
from ..createrepo import createrepo
from .builder import Builder, SrpmBuilder
# class BuilderThread(object):
# def __init__(self, builder_obj):
# self.builder = builder_obj
# self._running = False
#
# def terminate(self):
# self._running = False
#
# def run(self):
# self.builder.start_build()
# self._running = True
# state = None
# while self._running:
# state = self.builder.update_progress()
# if state in ["done", "failed"]:
# self._running = False
# else:
# time.sleep(5)
#
# if state == "done":
# self.builder.after_build()
[docs]class MockRemote(object):
# TODO: Refactor me!
# mock remote now do too much things
# idea: send events according to the build progress to handler
def __init__(self, builder_host, job, logger,
opts=None):
"""
:param builder_host: builder hostname or ip
:param backend.job.BuildJob job: Job object with the following attributes::
:ivar timeout: ssh timeout
:ivar destdir: target directory to put built packages
:ivar chroot: chroot config name/base to use in the mock build
(e.g.: fedora20_i386 )
:ivar build_id: copr build.id
:ivar pkg: pkg to build
:param macros: { "copr_username": ...,
"copr_projectname": ...,
"vendor": ...}
:param Munch opts: builder options, used keys::
:ivar build_user: user to run as/connect as on builder systems
:ivar do_sign: enable package signing, require configured
signer host and correct /etc/sign.conf
:ivar frontend_base_url: url to the copr frontend
:ivar results_baseurl: base url for the built results
# Removed:
# :param cont: if a pkg fails to build, continue to the next one--
# :param bool recurse: if more than one pkg and it fails to build,
# try to build the rest and come back to it
"""
self.opts = Munch(
do_sign=False,
frontend_base_url=None,
results_baseurl=u"",
build_user=DEF_BUILD_USER,
timeout=DEF_BUILD_TIMEOUT,
)
if opts:
self.opts.update(opts)
self.log = logger
self.job = job
self.log.info("Setting up builder: {0}".format(builder_host))
# TODO: add option "builder_log_level" to backend config
self.log.setLevel(logging.INFO)
builder_cls = (SrpmBuilder if self.job.chroot == 'srpm-builds' else Builder)
self.builder = builder_cls(
opts=self.opts,
hostname=builder_host,
job=self.job,
logger=logger,
)
self.failed = []
self.finished = []
[docs] def check(self):
"""
Checks that MockRemote configuration and environment are correct.
:raises MockRemoteError: when configuration is wrong or
some expected resource is unavailable
"""
if not self.job.chroot:
raise MockRemoteError("No chroot specified!")
try:
self.builder.check()
except BuilderError as error:
raise MockRemoteError(str(error))
@property
def chroot_dir(self):
return os.path.normpath(os.path.join(self.job.destdir, self.job.chroot))
@property
def pkg(self):
return self.job.pkg
[docs] def add_pubkey(self):
"""
Adds pubkey.gpg with public key to ``chroot_dir``
using `copr_username` and `copr_projectname` from self.job.
"""
self.log.info("Retrieving pubkey ")
# TODO: sign repodata as well ?
user = self.job.project_owner
project = self.job.project_name
pubkey_path = os.path.join(self.job.destdir, "pubkey.gpg")
try:
# TODO: uncomment this when key revoke/change will be implemented
# if os.path.exists(pubkey_path):
# return
get_pubkey(user, project, pubkey_path)
self.log.info(
"Added pubkey for user {} project {} into: {}".
format(user, project, pubkey_path))
except Exception as e:
self.log.exception(
"failed to retrieve pubkey for user {} project {} due to: \n"
"{}".format(user, project, e))
[docs] def sign_built_packages(self):
"""
Sign built rpms
using `copr_username` and `copr_projectname` from self.job
by means of obs-sign. If user builds doesn't have a key pair
at sign service, it would be created through ``copr-keygen``
:param chroot_dir: Directory with rpms to be signed
:param pkg: path to the source package
"""
self.log.info("Going to sign pkgs from source: {} in chroot: {}"
.format(self.job, self.chroot_dir))
try:
sign_rpms_in_dir(
self.job.project_owner,
self.job.project_name,
os.path.join(self.chroot_dir, self.job.target_dir_name),
opts=self.opts,
log=self.log
)
except Exception as e:
self.log.exception(
"failed to sign packages built from `{}` with error".format(self.job))
if isinstance(e, MockRemoteError):
raise
self.log.info("Sign done")
[docs] def do_createrepo(self):
if self.job.chroot == 'srpm-builds':
return
project_owner = self.job.project_owner
project_name = self.job.project_name
base_url = "/".join([self.opts.results_baseurl, project_owner,
project_name, self.job.chroot])
self.log.info("Createrepo:: owner: {}; project: {}; "
"front url: {}; path: {}; base_url: {}"
.format(project_owner, project_name,
self.opts.frontend_base_url, self.chroot_dir, base_url))
devel = uses_devel_repo(self.opts.frontend_base_url,
project_owner, project_name)
try:
createrepo(
path=self.chroot_dir,
base_url=base_url,
username=project_owner,
projectname=project_name,
devel=devel,
)
except CreateRepoError:
self.log.exception("Error making local repo: {}".format(self.chroot_dir))
# FIXME - maybe clean up .repodata and .olddata
# here?
[docs] def on_success_build(self):
self.log.info("Success building {0}".format(self.job.package_name))
if self.opts.do_sign:
self.sign_built_packages()
# createrepo with the new pkgs
self.do_createrepo()
[docs] def prepare_build_dir(self):
p_path = self.job.results_dir
# if it's marked as fail, nuke the failure and try to rebuild it
if os.path.exists(os.path.join(p_path, "fail")):
os.unlink(os.path.join(p_path, "fail"))
if os.path.exists(os.path.join(p_path, "success")):
os.unlink(os.path.join(p_path, "success"))
# mkdir to download results
if not os.path.exists(p_path):
os.makedirs(p_path)
self.mark_dir_with_build_id()
# def add_log_symlinks(self):
# # adding symlinks foo.log.gz -> foo.log for nginx web server auto index
# try:
# base = self._get_pkg_destpath()
# for name in os.listdir(base):
# if not name.endswith(".log.gz"):
# continue
# full_path = os.path.join(base, name)
# if os.path.isfile(full_path):
# os.symlink(full_path, full_path.replace(".log.gz", ".log"))
# except Exception as err:
# self.log.exception(err)
[docs] def build_pkg(self):
self.log.info("Start build: {}".format(self.job))
self.prepare_build_dir()
try:
self.builder.build()
except BuilderError as error:
self.log.error(str(error))
raise MockRemoteError("Builder error during job {}.".format(self.job))
[docs] def compress_live_log(self, job):
log_basename = "builder-live.log"
src = os.path.join(job.results_dir, log_basename)
# For automatic redirect from log to log.gz, consider configuring
# lighttpd like:
# url.rewrite-if-not-file = ("^/(.*)/builder-live.log$" => "/$1/redirect-live.log")
# url.redirect("^/(.*)/redirect-live.log$" => "/$1/builder-live.log.gz")
# or apache by:
# <Files builder-live.log>
# RewriteEngine on
# RewriteCond %{REQUEST_FILENAME} !-f
# RewriteRule ^(.*)$ %{REQUEST_URI}.gz [R]
# </files>
self.log.info("Compressing {} by gzip".format(src))
if subprocess.call(["gzip", src]) not in [0, 2]:
self.log.error("Unable to compress file {}".format(src))
return
[docs] def reattach_to_pkg_build(self):
"""
:raises VmError, BuilderError
"""
self.log.info("Reattach to build: {}".format(self.job))
try:
self.builder.reattach()
except BuilderError as error:
self.log.error(str(error))
raise MockRemoteError("Builder error during job {}.".format(self.job))
[docs] def check_build_success(self):
"""
Raise MockRemoteError if builder claims that the build failed.
"""
if not self.builder.check_build_success():
raise MockRemoteError("Build {} failed".format(self.job))
[docs] def download_results(self):
try:
self.builder.download_results(self.job.results_dir)
except Exception as err:
self.log.exception(err)
[docs] def mark_dir_with_build_id(self):
"""
Places "build.info" which contains job build_id
into the directory with downloaded files.
"""
info_file_path = os.path.join(self.job.results_dir, "build.info")
self.log.info("marking build dir with build_id, ")
try:
with open(info_file_path, 'w') as info_file:
info_file.writelines([
"build_id={}".format(self.job.build_id),
"\nbuilder_ip={}".format(self.builder.hostname)])
except Exception as error:
self.log.exception("Failed to mark build {} with build_id".format(error))