Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import flask 
   8  import sqlite3 
   9  import requests 
  10   
  11  from flask import request 
  12  from sqlalchemy.sql import text 
  13  from sqlalchemy import or_ 
  14  from sqlalchemy import and_ 
  15  from sqlalchemy import func 
  16  from sqlalchemy.orm import joinedload 
  17  from sqlalchemy.orm.exc import NoResultFound 
  18  from sqlalchemy.sql import false,true 
  19  from werkzeug.utils import secure_filename 
  20  from sqlalchemy import desc, asc, bindparam, Integer, String 
  21  from collections import defaultdict 
  22   
  23  from coprs import app 
  24  from coprs import db 
  25  from coprs import exceptions 
  26  from coprs import models 
  27  from coprs import helpers 
  28  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  29  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException 
  30  from coprs.helpers import StatusEnum 
  31   
  32  from coprs.logic import coprs_logic 
  33  from coprs.logic import users_logic 
  34  from coprs.logic.actions_logic import ActionsLogic 
  35  from coprs.models import BuildChroot,Build,Package,MockChroot 
  36  from .coprs_logic import MockChrootsLogic 
  37   
  38  log = app.logger 
39 40 41 -class BuildsLogic(object):
42 @classmethod
43 - def get(cls, build_id):
44 return models.Build.query.filter(models.Build.id == build_id)
45 46 @classmethod
47 - def get_build_tasks(cls, status, background=None):
48 """ Returns tasks with given status. If background is specified then 49 returns normal jobs (false) or background jobs (true) 50 """ 51 result = models.BuildChroot.query.join(models.Build)\ 52 .filter(models.BuildChroot.status == status)\ 53 .order_by(models.Build.id.asc()) 54 if background is not None: 55 result = result.filter(models.Build.is_background == (true() if background else false())) 56 return result
57 58 @classmethod
59 - def get_srpm_build_tasks(cls, status, background=None):
60 """ Returns srpm build tasks with given status. If background is 61 specified then returns normal jobs (false) or background jobs (true) 62 """ 63 result = models.Build.query\ 64 .filter(models.Build.source_status == status)\ 65 .order_by(models.Build.id.asc()) 66 if background is not None: 67 result = result.filter(models.Build.is_background == (true() if background else false())) 68 return result
69 70 @classmethod
71 - def get_recent_tasks(cls, user=None, limit=None):
72 if not limit: 73 limit = 100 74 75 query = models.Build.query 76 if user is not None: 77 query = query.filter(models.Build.user_id == user.id) 78 79 query = query.join( 80 models.BuildChroot.query 81 .filter(models.BuildChroot.ended_on.isnot(None)) 82 .order_by(models.BuildChroot.ended_on.desc()) 83 .subquery() 84 ).order_by(models.Build.id.desc()) 85 86 # Workaround - otherwise it could take less records than `limit` even though there are more of them. 87 query = query.limit(limit if limit > 100 else 100) 88 return list(query.all()[:4])
89 90 @classmethod
91 - def get_running_tasks_by_time(cls, start, end):
92 result = models.BuildChroot.query\ 93 .filter(models.BuildChroot.ended_on > start)\ 94 .filter(models.BuildChroot.started_on < end)\ 95 .order_by(models.BuildChroot.started_on.asc()) 96 97 return result
98 99 @classmethod
101 end = int(time.time()) 102 start = end - 86399 103 step = 3600 104 tasks = cls.get_running_tasks_by_time(start, end) 105 steps = int(round((end - start) / step + 0.5)) 106 current_step = 0 107 108 data = [[0] * (steps + 1)] 109 data[0][0] = '' 110 for t in tasks: 111 task = t.to_dict() 112 while task['started_on'] > start + step * (current_step + 1): 113 current_step += 1 114 data[0][current_step + 1] += 1 115 return data
116 117 @classmethod
118 - def get_chroot_histogram(cls, start, end):
119 chroots = [] 120 chroot_query = BuildChroot.query\ 121 .filter(models.BuildChroot.started_on < end)\ 122 .filter(models.BuildChroot.ended_on > start)\ 123 .with_entities(BuildChroot.mock_chroot_id, 124 func.count(BuildChroot.mock_chroot_id))\ 125 .group_by(BuildChroot.mock_chroot_id)\ 126 .order_by(BuildChroot.mock_chroot_id) 127 128 for chroot in chroot_query: 129 chroots.append([chroot[0], chroot[1]]) 130 131 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 132 for mock_chroot in mock_chroots: 133 for l in chroots: 134 if l[0] == mock_chroot.id: 135 l[0] = mock_chroot.name 136 137 return chroots
138 139 @classmethod
140 - def get_tasks_histogram(cls, type, start, end, step):
141 start = start - (start % step) # align graph interval to a multiple of step 142 end = end - (end % step) 143 steps = int((end - start) / step + 0.5) 144 data = [['pending'], ['running'], ['avg running'], ['time']] 145 146 result = models.BuildsStatistics.query\ 147 .filter(models.BuildsStatistics.stat_type == type)\ 148 .filter(models.BuildsStatistics.time >= start)\ 149 .filter(models.BuildsStatistics.time <= end)\ 150 .order_by(models.BuildsStatistics.time) 151 152 for row in result: 153 data[0].append(row.pending) 154 data[1].append(row.running) 155 156 for i in range(len(data[0]) - 1, steps): 157 step_start = start + i * step 158 step_end = step_start + step 159 160 query_pending = text(""" 161 SELECT COUNT(*) as pending 162 FROM build_chroot JOIN build on build.id = build_chroot.build_id 163 WHERE 164 build.submitted_on < :end 165 AND ( 166 build_chroot.started_on > :start 167 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 168 -- for currently pending builds we need to filter on status=pending because there might be 169 -- failed builds that have started_on=NULL 170 ) 171 AND NOT build.canceled 172 """) 173 174 query_running = text(""" 175 SELECT COUNT(*) as running 176 FROM build_chroot 177 WHERE 178 started_on < :end 179 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 180 -- for currently running builds we need to filter on status=running because there might be failed 181 -- builds that have ended_on=NULL 182 """) 183 184 res_pending = db.engine.execute(query_pending, start=step_start, end=step_end, 185 status=helpers.StatusEnum('pending')) 186 res_running = db.engine.execute(query_running, start=step_start, end=step_end, 187 status=helpers.StatusEnum('running')) 188 189 pending = res_pending.first().pending 190 running = res_running.first().running 191 data[0].append(pending) 192 data[1].append(running) 193 194 statistic = models.BuildsStatistics( 195 time = step_start, 196 stat_type = type, 197 running = running, 198 pending = pending 199 ) 200 db.session.merge(statistic) 201 db.session.commit() 202 203 running_total = 0 204 for i in range(1, steps + 1): 205 running_total += data[1][i] 206 207 data[2].extend([running_total * 1.0 / steps] * (len(data[0]) - 1)) 208 209 for i in range(start, end, step): 210 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 211 212 return data
213 214 @classmethod
215 - def get_build_importing_queue(cls, background=None):
216 """ 217 Returns Builds which are waiting to be uploaded to dist git 218 """ 219 query = (models.Build.query 220 .filter(models.Build.canceled == false()) 221 .filter(models.Build.source_status == helpers.StatusEnum("importing")) 222 .order_by(models.Build.id.asc())) 223 if background is not None: 224 query = query.filter(models.Build.is_background == (true() if background else false())) 225 return query
226 227 @classmethod
228 - def get_pending_srpm_build_tasks(cls, background=None):
229 query = (models.Build.query 230 .filter(models.Build.canceled == false()) 231 .filter(models.Build.source_status == helpers.StatusEnum("pending")) 232 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 233 if background is not None: 234 query = query.filter(models.Build.is_background == (true() if background else false())) 235 return query
236 237 @classmethod
238 - def get_pending_build_tasks(cls, background=None):
239 query = (models.BuildChroot.query.join(models.Build) 240 .filter(models.Build.canceled == false()) 241 .filter(or_( 242 models.BuildChroot.status == helpers.StatusEnum("pending"), 243 and_( 244 models.BuildChroot.status == helpers.StatusEnum("running"), 245 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 246 models.BuildChroot.ended_on.is_(None) 247 ) 248 )) 249 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 250 if background is not None: 251 query = query.filter(models.Build.is_background == (true() if background else false())) 252 return query
253 254 @classmethod
255 - def get_build_task(cls, task_id):
256 try: 257 build_id, chroot_name = task_id.split("-", 1) 258 except ValueError: 259 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 260 261 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 262 return build_chroot.join(models.Build).first()
263 264 @classmethod
265 - def get_srpm_build_task(cls, build_id):
266 return BuildsLogic.get_by_id(build_id).first()
267 268 @classmethod
269 - def get_multiple(cls):
270 return models.Build.query.order_by(models.Build.id.desc())
271 272 @classmethod
273 - def get_multiple_by_copr(cls, copr):
274 """ Get collection of builds in copr sorted by build_id descending 275 """ 276 return cls.get_multiple().filter(models.Build.copr == copr)
277 278 @classmethod
279 - def get_multiple_by_user(cls, user):
280 """ Get collection of builds in copr sorted by build_id descending 281 form the copr belonging to `user` 282 """ 283 return cls.get_multiple().join(models.Build.copr).filter( 284 models.Copr.user == user)
285 286 @classmethod
287 - def init_db(cls):
288 if db.engine.url.drivername == "sqlite": 289 return 290 291 status_to_order = """ 292 CREATE OR REPLACE FUNCTION status_to_order (x integer) 293 RETURNS integer AS $$ BEGIN 294 RETURN CASE WHEN x = 3 THEN 1 295 WHEN x = 6 THEN 2 296 WHEN x = 7 THEN 3 297 WHEN x = 4 THEN 4 298 WHEN x = 0 THEN 5 299 WHEN x = 1 THEN 6 300 WHEN x = 5 THEN 7 301 WHEN x = 2 THEN 8 302 WHEN x = 8 THEN 9 303 WHEN x = 9 THEN 10 304 ELSE x 305 END; END; 306 $$ LANGUAGE plpgsql; 307 """ 308 309 order_to_status = """ 310 CREATE OR REPLACE FUNCTION order_to_status (x integer) 311 RETURNS integer AS $$ BEGIN 312 RETURN CASE WHEN x = 1 THEN 3 313 WHEN x = 2 THEN 6 314 WHEN x = 3 THEN 7 315 WHEN x = 4 THEN 4 316 WHEN x = 5 THEN 0 317 WHEN x = 6 THEN 1 318 WHEN x = 7 THEN 5 319 WHEN x = 8 THEN 2 320 WHEN x = 9 THEN 8 321 WHEN x = 10 THEN 9 322 ELSE x 323 END; END; 324 $$ LANGUAGE plpgsql; 325 """ 326 327 db.engine.connect() 328 db.engine.execute(status_to_order) 329 db.engine.execute(order_to_status)
330 331 @classmethod
332 - def get_copr_builds_list(cls, copr, dirname=''):
333 query_select = """ 334 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 335 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 336 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 337 FROM build 338 LEFT OUTER JOIN package 339 ON build.package_id = package.id 340 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 341 ON statuses.build_id=build.id 342 LEFT OUTER JOIN copr 343 ON copr.id = build.copr_id 344 LEFT OUTER JOIN copr_dir 345 ON build.copr_dir_id = copr_dir.id 346 LEFT OUTER JOIN "user" 347 ON copr.user_id = "user".id 348 LEFT OUTER JOIN "group" 349 ON copr.group_id = "group".id 350 WHERE build.copr_id = :copr_id 351 AND (:dirname = '' OR :dirname = copr_dir.name) 352 GROUP BY 353 build.id; 354 """ 355 356 if db.engine.url.drivername == "sqlite": 357 def sqlite_status_to_order(x): 358 if x == 3: 359 return 1 360 elif x == 6: 361 return 2 362 elif x == 7: 363 return 3 364 elif x == 4: 365 return 4 366 elif x == 0: 367 return 5 368 elif x == 1: 369 return 6 370 elif x == 5: 371 return 7 372 elif x == 2: 373 return 8 374 elif x == 8: 375 return 9 376 elif x == 9: 377 return 10 378 return 1000
379 380 def sqlite_order_to_status(x): 381 if x == 1: 382 return 3 383 elif x == 2: 384 return 6 385 elif x == 3: 386 return 7 387 elif x == 4: 388 return 4 389 elif x == 5: 390 return 0 391 elif x == 6: 392 return 1 393 elif x == 7: 394 return 5 395 elif x == 8: 396 return 2 397 elif x == 9: 398 return 8 399 elif x == 10: 400 return 9 401 return 1000
402 403 conn = db.engine.connect() 404 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 405 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 406 statement = text(query_select) 407 statement.bindparams(bindparam("copr_id", Integer)) 408 statement.bindparams(bindparam("dirname", String)) 409 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 410 else: 411 statement = text(query_select) 412 statement.bindparams(bindparam("copr_id", Integer)) 413 statement.bindparams(bindparam("dirname", String)) 414 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 415 416 return result 417 418 @classmethod
419 - def join_group(cls, query):
420 return query.join(models.Copr).outerjoin(models.Group)
421 422 @classmethod
423 - def get_multiple_by_name(cls, username, coprname):
424 query = cls.get_multiple() 425 return (query.join(models.Build.copr) 426 .options(db.contains_eager(models.Build.copr)) 427 .join(models.Copr.user) 428 .filter(models.Copr.name == coprname) 429 .filter(models.User.username == username))
430 431 @classmethod
432 - def get_by_ids(cls, ids):
433 return models.Build.query.filter(models.Build.id.in_(ids))
434 435 @classmethod
436 - def get_by_id(cls, build_id):
437 return models.Build.query.filter(models.Build.id == build_id)
438 439 @classmethod
440 - def create_new_from_other_build(cls, user, copr, source_build, 441 chroot_names=None, **build_options):
442 skip_import = False 443 git_hashes = {} 444 445 if source_build.source_type == helpers.BuildSourceEnum('upload'): 446 if source_build.repeatable: 447 skip_import = True 448 for chroot in source_build.build_chroots: 449 git_hashes[chroot.name] = chroot.git_hash 450 else: 451 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 452 453 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 454 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 455 srpm_url=source_build.srpm_url, **build_options) 456 build.package_id = source_build.package_id 457 build.pkg_version = source_build.pkg_version 458 return build
459 460 @classmethod
461 - def create_new_from_url(cls, user, copr, url, 462 chroot_names=None, **build_options):
463 """ 464 :type user: models.User 465 :type copr: models.Copr 466 467 :type chroot_names: List[str] 468 469 :rtype: models.Build 470 """ 471 source_type = helpers.BuildSourceEnum("link") 472 source_json = json.dumps({"url": url}) 473 srpm_url = None if url.endswith('.spec') else url 474 return cls.create_new(user, copr, source_type, source_json, chroot_names, 475 pkgs=url, srpm_url=srpm_url, **build_options)
476 477 @classmethod
478 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 479 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 480 chroot_names=None, **build_options):
481 """ 482 :type user: models.User 483 :type copr: models.Copr 484 485 :type chroot_names: List[str] 486 487 :rtype: models.Build 488 """ 489 source_type = helpers.BuildSourceEnum("scm") 490 source_json = json.dumps({"type": scm_type, 491 "clone_url": clone_url, 492 "committish": committish, 493 "subdirectory": subdirectory, 494 "spec": spec, 495 "srpm_build_method": srpm_build_method}) 496 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
497 498 @classmethod
499 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions, 500 chroot_names=None, **build_options):
501 """ 502 :type user: models.User 503 :type copr: models.Copr 504 :type package_name: str 505 :type version: str 506 :type python_versions: List[str] 507 508 :type chroot_names: List[str] 509 510 :rtype: models.Build 511 """ 512 source_type = helpers.BuildSourceEnum("pypi") 513 source_json = json.dumps({"pypi_package_name": pypi_package_name, 514 "pypi_package_version": pypi_package_version, 515 "python_versions": python_versions}) 516 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
517 518 @classmethod
519 - def create_new_from_rubygems(cls, user, copr, gem_name, 520 chroot_names=None, **build_options):
521 """ 522 :type user: models.User 523 :type copr: models.Copr 524 :type gem_name: str 525 :type chroot_names: List[str] 526 :rtype: models.Build 527 """ 528 source_type = helpers.BuildSourceEnum("rubygems") 529 source_json = json.dumps({"gem_name": gem_name}) 530 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
531 532 @classmethod
533 - def create_new_from_custom(cls, user, copr, 534 script, script_chroot=None, script_builddeps=None, 535 script_resultdir=None, chroot_names=None, **kwargs):
536 """ 537 :type user: models.User 538 :type copr: models.Copr 539 :type script: str 540 :type script_chroot: str 541 :type script_builddeps: str 542 :type script_resultdir: str 543 :type chroot_names: List[str] 544 :rtype: models.Build 545 """ 546 source_type = helpers.BuildSourceEnum("custom") 547 source_dict = { 548 'script': script, 549 'chroot': script_chroot, 550 'builddeps': script_builddeps, 551 'resultdir': script_resultdir, 552 } 553 554 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 555 chroot_names, **kwargs)
556 557 @classmethod
558 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 559 chroot_names=None, **build_options):
560 """ 561 :type user: models.User 562 :type copr: models.Copr 563 :param f_uploader(file_path): function which stores data at the given `file_path` 564 :return: 565 """ 566 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 567 tmp_name = os.path.basename(tmp) 568 filename = secure_filename(orig_filename) 569 file_path = os.path.join(tmp, filename) 570 f_uploader(file_path) 571 572 # make the pkg public 573 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 574 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 575 tmp_dir=tmp_name, 576 filename=filename) 577 578 # create json describing the build source 579 source_type = helpers.BuildSourceEnum("upload") 580 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 581 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 582 583 try: 584 build = cls.create_new(user, copr, source_type, source_json, 585 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options) 586 except Exception: 587 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 588 raise 589 590 return build
591 592 @classmethod
593 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 594 git_hashes=None, skip_import=False, background=False, batch=None, 595 srpm_url=None, **build_options):
596 """ 597 :type user: models.User 598 :type copr: models.Copr 599 :type chroot_names: List[str] 600 :type source_type: int value from helpers.BuildSourceEnum 601 :type source_json: str in json format 602 :type pkgs: str 603 :type git_hashes: dict 604 :type skip_import: bool 605 :type background: bool 606 :type batch: models.Batch 607 :rtype: models.Build 608 """ 609 if chroot_names is None: 610 chroots = [c for c in copr.active_chroots] 611 else: 612 chroots = [] 613 for chroot in copr.active_chroots: 614 if chroot.name in chroot_names: 615 chroots.append(chroot) 616 617 build = cls.add( 618 user=user, 619 pkgs=pkgs, 620 copr=copr, 621 chroots=chroots, 622 source_type=source_type, 623 source_json=source_json, 624 enable_net=build_options.get("enable_net", copr.build_enable_net), 625 background=background, 626 git_hashes=git_hashes, 627 skip_import=skip_import, 628 batch=batch, 629 srpm_url=srpm_url, 630 ) 631 632 if user.proven: 633 if "timeout" in build_options: 634 build.timeout = build_options["timeout"] 635 636 return build
637 638 @classmethod
639 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 640 repos=None, chroots=None, timeout=None, enable_net=True, 641 git_hashes=None, skip_import=False, background=False, batch=None, 642 srpm_url=None):
643 644 if chroots is None: 645 chroots = [] 646 647 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 648 copr, "Can't build while there is an operation in progress: {action}") 649 users_logic.UsersLogic.raise_if_cant_build_in_copr( 650 user, copr, 651 "You don't have permissions to build in this copr.") 652 653 if not repos: 654 repos = copr.repos 655 656 # todo: eliminate pkgs and this check 657 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 658 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg " 659 "with bad characters. Forgot to split?") 660 661 # just temporary to keep compatibility 662 if not source_type or not source_json: 663 source_type = helpers.BuildSourceEnum("link") 664 source_json = json.dumps({"url":pkgs}) 665 666 if skip_import and srpm_url: 667 chroot_status = StatusEnum("pending") 668 source_status = StatusEnum("succeeded") 669 elif srpm_url: 670 chroot_status = StatusEnum("waiting") 671 source_status = StatusEnum("importing") 672 else: 673 chroot_status = StatusEnum("waiting") 674 source_status = StatusEnum("pending") 675 676 build = models.Build( 677 user=user, 678 pkgs=pkgs, 679 copr=copr, 680 repos=repos, 681 source_type=source_type, 682 source_json=source_json, 683 source_status=source_status, 684 submitted_on=int(time.time()), 685 enable_net=bool(enable_net), 686 is_background=bool(background), 687 batch=batch, 688 srpm_url=srpm_url, 689 ) 690 691 if timeout: 692 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 693 694 db.session.add(build) 695 696 # add BuildChroot object for each active (or selected) chroot 697 # this copr is assigned to 698 if not chroots: 699 chroots = copr.active_chroots 700 701 for chroot in chroots: 702 git_hash = None 703 if git_hashes: 704 git_hash = git_hashes.get(chroot.name) 705 buildchroot = models.BuildChroot( 706 build=build, 707 status=chroot_status, 708 mock_chroot=chroot, 709 git_hash=git_hash, 710 ) 711 db.session.add(buildchroot) 712 713 return build
714 715 @classmethod
716 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 717 scm_object_type=None, scm_object_id=None, scm_object_url=None):
718 719 source_dict = package.source_json_dict 720 source_dict.update(source_dict_update) 721 source_json = json.dumps(source_dict) 722 723 if not copr_dir: 724 copr_dir = package.copr.main_dir 725 726 build = models.Build( 727 user=None, 728 pkgs=None, 729 package=package, 730 copr=package.copr, 731 repos=package.copr.repos, 732 source_status=helpers.StatusEnum("pending"), 733 source_type=package.source_type, 734 source_json=source_json, 735 submitted_on=int(time.time()), 736 enable_net=package.copr.build_enable_net, 737 timeout=DEFAULT_BUILD_TIMEOUT, 738 copr_dir=copr_dir, 739 update_callback=update_callback, 740 scm_object_type=scm_object_type, 741 scm_object_id=scm_object_id, 742 scm_object_url=scm_object_url, 743 ) 744 db.session.add(build) 745 746 chroots = package.copr.active_chroots 747 status = helpers.StatusEnum("waiting") 748 for chroot in chroots: 749 buildchroot = models.BuildChroot( 750 build=build, 751 status=status, 752 mock_chroot=chroot, 753 git_hash=None 754 ) 755 db.session.add(buildchroot) 756 757 cls.process_update_callback(build) 758 return build
759 760 761 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 762 763 @classmethod
764 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
765 """ 766 Returns a list of BuildChroots identified by build_id and dist-git 767 branch name. 768 """ 769 return ( 770 models.BuildChroot.query 771 .join(models.MockChroot) 772 .filter(models.BuildChroot.build_id==build_id) 773 .filter(models.MockChroot.distgit_branch_name==branch) 774 ).all()
775 776 777 @classmethod
778 - def delete_local_source(cls, build):
779 """ 780 Deletes the locally stored data for build purposes. This is typically 781 uploaded srpm file, uploaded spec file or webhook POST content. 782 """ 783 # is it hosted on the copr frontend? 784 data = json.loads(build.source_json) 785 if 'tmp' in data: 786 tmp = data["tmp"] 787 storage_path = app.config["STORAGE_DIR"] 788 try: 789 shutil.rmtree(os.path.join(storage_path, tmp)) 790 except: 791 pass
792 793 794 @classmethod
795 - def update_state_from_dict(cls, build, upd_dict):
796 """ 797 :param build: 798 :param upd_dict: 799 example: 800 { 801 "builds":[ 802 { 803 "id": 1, 804 "copr_id": 2, 805 "started_on": 1390866440 806 }, 807 { 808 "id": 2, 809 "copr_id": 1, 810 "status": 0, 811 "chroot": "fedora-18-x86_64", 812 "result_dir": "baz", 813 "ended_on": 1390866440 814 }] 815 } 816 """ 817 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 818 819 # update build 820 for attr in ["built_packages", "srpm_url"]: 821 value = upd_dict.get(attr, None) 822 if value: 823 setattr(build, attr, value) 824 825 # update source build status 826 if upd_dict.get("task_id") == build.task_id: 827 build.result_dir = upd_dict.get("result_dir", "") 828 829 if upd_dict.get("status") == StatusEnum("succeeded"): 830 new_status = StatusEnum("importing") 831 else: 832 new_status = upd_dict.get("status") 833 834 build.source_status = new_status 835 if new_status == StatusEnum("failed") or \ 836 new_status == StatusEnum("skipped"): 837 for ch in build.build_chroots: 838 ch.status = new_status 839 ch.ended_on = upd_dict.get("ended_on") or time.time() 840 db.session.add(ch) 841 842 if new_status == StatusEnum("failed"): 843 build.fail_type = helpers.FailTypeEnum("srpm_build_error") 844 845 cls.process_update_callback(build) 846 db.session.add(build) 847 return 848 849 if "chroot" in upd_dict: 850 # update respective chroot status 851 for build_chroot in build.build_chroots: 852 if build_chroot.name == upd_dict["chroot"]: 853 build_chroot.result_dir = upd_dict.get("result_dir", "") 854 855 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 856 build_chroot.status = upd_dict["status"] 857 858 if upd_dict.get("status") in BuildsLogic.terminal_states: 859 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 860 861 if upd_dict.get("status") == StatusEnum("starting"): 862 build_chroot.started_on = upd_dict.get("started_on") or time.time() 863 864 db.session.add(build_chroot) 865 866 # If the last package of a module was successfully built, 867 # then send an action to create module repodata on backend 868 if (build.module 869 and upd_dict.get("status") == StatusEnum("succeeded") 870 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 871 ActionsLogic.send_build_module(build.copr, build.module) 872 873 cls.process_update_callback(build) 874 db.session.add(build)
875 876 @classmethod
877 - def process_update_callback(cls, build):
878 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 879 if not parsed_git_url: 880 return 881 882 if build.update_callback == 'pagure_flag_pull_request': 883 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 884 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 885 return cls.pagure_flag(build, api_url) 886 887 elif build.update_callback == 'pagure_flag_commit': 888 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 889 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 890 return cls.pagure_flag(build, api_url)
891 892 @classmethod
893 - def pagure_flag(cls, build, api_url):
894 headers = { 895 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 896 } 897 898 if build.srpm_url: 899 progress = 50 900 else: 901 progress = 10 902 903 state_table = { 904 'failed': ('failure', 0), 905 'succeeded': ('success', 100), 906 'canceled': ('canceled', 0), 907 'running': ('pending', progress), 908 'pending': ('pending', progress), 909 'skipped': ('error', 0), 910 'starting': ('pending', progress), 911 'importing': ('pending', progress), 912 'forked': ('error', 0), 913 'waiting': ('pending', progress), 914 'unknown': ('error', 0), 915 } 916 917 build_url = os.path.join( 918 app.config['PUBLIC_COPR_BASE_URL'], 919 'coprs', build.copr.full_name.replace('@', 'g/'), 920 'build', str(build.id) 921 ) 922 923 data = { 924 'username': 'Copr build', 925 'comment': '#{}'.format(build.id), 926 'url': build_url, 927 'status': state_table[build.state][0], 928 'percent': state_table[build.state][1], 929 'uid': str(build.id), 930 } 931 932 log.info('Sending data to Pagure API: %s', pprint.pformat(data)) 933 response = requests.post(api_url, data=data, headers=headers) 934 log.info('Pagure API response: %s', response.text)
935 936 @classmethod
937 - def cancel_build(cls, user, build):
938 if not user.can_build_in(build.copr): 939 raise exceptions.InsufficientRightsException( 940 "You are not allowed to cancel this build.") 941 if not build.cancelable: 942 if build.status == StatusEnum("starting"): 943 # this is not intuitive, that's why we provide more specific message 944 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 945 else: 946 err_msg = "Cannot cancel build {}".format(build.id) 947 raise exceptions.RequestCannotBeExecuted(err_msg) 948 949 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 950 ActionsLogic.send_cancel_build(build) 951 952 build.canceled = True 953 cls.process_update_callback(build) 954 955 for chroot in build.build_chroots: 956 chroot.status = 2 # canceled 957 if chroot.ended_on is not None: 958 chroot.ended_on = time.time()
959 960 @classmethod
961 - def delete_build(cls, user, build, send_delete_action=True):
962 """ 963 :type user: models.User 964 :type build: models.Build 965 """ 966 if not user.can_edit(build.copr) or build.persistent: 967 raise exceptions.InsufficientRightsException( 968 "You are not allowed to delete build `{}`.".format(build.id)) 969 970 if not build.finished: 971 raise exceptions.ActionInProgressException( 972 "You can not delete build `{}` which is not finished.".format(build.id), 973 "Unfinished build") 974 975 if send_delete_action: 976 ActionsLogic.send_delete_build(build) 977 978 for build_chroot in build.build_chroots: 979 db.session.delete(build_chroot) 980 981 db.session.delete(build)
982 983 @classmethod
984 - def mark_as_failed(cls, build_id):
985 """ 986 Marks build as failed on all its non-finished chroots 987 """ 988 build = cls.get(build_id).one() 989 chroots = filter(lambda x: x.status != helpers.StatusEnum("succeeded"), build.build_chroots) 990 for chroot in chroots: 991 chroot.status = helpers.StatusEnum("failed") 992 cls.process_update_callback(build) 993 return build
994 995 @classmethod
996 - def last_modified(cls, copr):
997 """ Get build datetime (as epoch) of last successful build 998 999 :arg copr: object of copr 1000 """ 1001 builds = cls.get_multiple_by_copr(copr) 1002 1003 last_build = ( 1004 builds.join(models.BuildChroot) 1005 .filter((models.BuildChroot.status == helpers.StatusEnum("succeeded")) 1006 | (models.BuildChroot.status == helpers.StatusEnum("skipped"))) 1007 .filter(models.BuildChroot.ended_on.isnot(None)) 1008 .order_by(models.BuildChroot.ended_on.desc()) 1009 ).first() 1010 if last_build: 1011 return last_build.ended_on 1012 else: 1013 return None
1014 1015 @classmethod
1016 - def filter_is_finished(cls, query, is_finished):
1017 # todo: check that ended_on is set correctly for all cases 1018 # e.g.: failed dist-git import, cancellation 1019 if is_finished: 1020 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1021 else: 1022 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1023 1024 @classmethod
1025 - def filter_by_group_name(cls, query, group_name):
1026 return query.filter(models.Group.name == group_name)
1027 1028 @classmethod
1029 - def filter_by_package_name(cls, query, package_name):
1030 return query.join(models.Package).filter(models.Package.name == package_name)
1031
1032 1033 -class BuildChrootsLogic(object):
1034 @classmethod
1035 - def get_by_build_id_and_name(cls, build_id, name):
1036 mc = MockChrootsLogic.get_from_name(name).one() 1037 1038 return ( 1039 BuildChroot.query 1040 .filter(BuildChroot.build_id == build_id) 1041 .filter(BuildChroot.mock_chroot_id == mc.id) 1042 )
1043 1044 @classmethod
1045 - def get_multiply(cls):
1046 query = ( 1047 models.BuildChroot.query 1048 .join(models.BuildChroot.build) 1049 .join(models.BuildChroot.mock_chroot) 1050 .join(models.Build.copr) 1051 .join(models.Copr.user) 1052 .outerjoin(models.Group) 1053 ) 1054 return query
1055 1056 @classmethod
1057 - def filter_by_build_id(cls, query, build_id):
1058 return query.filter(models.Build.id == build_id)
1059 1060 @classmethod
1061 - def filter_by_project_id(cls, query, project_id):
1062 return query.filter(models.Copr.id == project_id)
1063 1064 @classmethod
1065 - def filter_by_project_user_name(cls, query, username):
1066 return query.filter(models.User.username == username)
1067 1068 @classmethod
1069 - def filter_by_state(cls, query, state):
1071 1072 @classmethod
1073 - def filter_by_group_name(cls, query, group_name):
1074 return query.filter(models.Group.name == group_name)
1075
1076 1077 -class BuildsMonitorLogic(object):
1078 @classmethod
1079 - def get_monitor_data(cls, copr):
1080 query = """ 1081 SELECT 1082 package.id as package_id, 1083 package.name AS package_name, 1084 build.id AS build_id, 1085 build_chroot.status AS build_chroot_status, 1086 build.pkg_version AS build_pkg_version, 1087 mock_chroot.id AS mock_chroot_id, 1088 mock_chroot.os_release AS mock_chroot_os_release, 1089 mock_chroot.os_version AS mock_chroot_os_version, 1090 mock_chroot.arch AS mock_chroot_arch 1091 FROM package 1092 JOIN (SELECT 1093 MAX(build.id) AS max_build_id_for_chroot, 1094 build.package_id AS package_id, 1095 build_chroot.mock_chroot_id AS mock_chroot_id 1096 FROM build 1097 JOIN build_chroot 1098 ON build.id = build_chroot.build_id 1099 WHERE build.copr_id = {copr_id} 1100 AND build_chroot.status != 2 1101 GROUP BY build.package_id, 1102 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1103 ON package.id = max_build_ids_for_a_chroot.package_id 1104 JOIN build 1105 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1106 JOIN build_chroot 1107 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1108 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1109 JOIN mock_chroot 1110 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1111 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1112 """.format(copr_id=copr.id) 1113 rows = db.session.execute(query) 1114 return rows
1115