1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 from sqlalchemy.sql import text
10 from sqlalchemy import or_
11 from sqlalchemy import and_
12 from sqlalchemy.orm import joinedload
13 from sqlalchemy.orm.exc import NoResultFound
14 from sqlalchemy.sql import false,true
15 from werkzeug.utils import secure_filename
16 from sqlalchemy import desc,asc, bindparam, Integer
17 from collections import defaultdict
18
19 from coprs import app
20 from coprs import db
21 from coprs import exceptions
22 from coprs import models
23 from coprs import helpers
24 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT, MAX_PRIO
25 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException
26 from coprs.helpers import StatusEnum
27
28 from coprs.logic import coprs_logic
29 from coprs.logic import users_logic
30 from coprs.logic.actions_logic import ActionsLogic
31 from coprs.models import BuildChroot,Build,Package,MockChroot
32 from .coprs_logic import MockChrootsLogic
33
34 log = app.logger
119
120
121 @classmethod
128
129 @classmethod
135
136 @classmethod
141
142 @classmethod
155
156 @classmethod
162
163 @classmethod
168
169 @classmethod
174
175 @classmethod
184
185 @classmethod
188
189 @classmethod
190 - def defer(cls, build_id):
197
198 @classmethod
201
202 @classmethod
207
208 @classmethod
215
216
217 @classmethod
219 if db.engine.url.drivername == "sqlite":
220 return
221
222 status_to_order = """
223 CREATE OR REPLACE FUNCTION status_to_order (x integer)
224 RETURNS integer AS $$ BEGIN
225 RETURN CASE WHEN x = 0 THEN 0
226 WHEN x = 3 THEN 1
227 WHEN x = 6 THEN 2
228 WHEN x = 7 THEN 3
229 WHEN x = 4 THEN 4
230 WHEN x = 1 THEN 5
231 WHEN x = 5 THEN 6
232 ELSE 1000
233 END; END;
234 $$ LANGUAGE plpgsql;
235 """
236
237 order_to_status = """
238 CREATE OR REPLACE FUNCTION order_to_status (x integer)
239 RETURNS integer AS $$ BEGIN
240 RETURN CASE WHEN x = 0 THEN 0
241 WHEN x = 1 THEN 3
242 WHEN x = 2 THEN 6
243 WHEN x = 3 THEN 7
244 WHEN x = 4 THEN 4
245 WHEN x = 5 THEN 1
246 WHEN x = 6 THEN 5
247 ELSE 1000
248 END; END;
249 $$ LANGUAGE plpgsql;
250 """
251
252 db.engine.connect()
253 db.engine.execute(status_to_order)
254 db.engine.execute(order_to_status)
255
256 @classmethod
258 query_select = """
259 SELECT build.id, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
260 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
261 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name
262 FROM build
263 LEFT OUTER JOIN package
264 ON build.package_id = package.id
265 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
266 ON statuses.build_id=build.id
267 LEFT OUTER JOIN copr
268 ON copr.id = build.copr_id
269 LEFT OUTER JOIN "user"
270 ON copr.user_id = "user".id
271 LEFT OUTER JOIN "group"
272 ON copr.group_id = "group".id
273 WHERE build.copr_id = :copr_id
274 GROUP BY
275 build.id;
276 """
277
278 if db.engine.url.drivername == "sqlite":
279 def sqlite_status_to_order(x):
280 if x == 3:
281 return 1
282 elif x == 6:
283 return 2
284 elif x == 7:
285 return 3
286 elif x == 4:
287 return 4
288 elif x == 0:
289 return 5
290 elif x == 1:
291 return 6
292 elif x == 5:
293 return 7
294 elif x == 8:
295 return 8
296 return 1000
297
298 def sqlite_order_to_status(x):
299 if x == 1:
300 return 3
301 elif x == 2:
302 return 6
303 elif x == 3:
304 return 7
305 elif x == 4:
306 return 4
307 elif x == 5:
308 return 0
309 elif x == 6:
310 return 1
311 elif x == 7:
312 return 5
313 elif x == 8:
314 return 8
315 return 1000
316
317 conn = db.engine.connect()
318 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
319 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
320 statement = text(query_select)
321 statement.bindparams(bindparam("copr_id", Integer))
322 result = conn.execute(statement, {"copr_id": copr.id})
323 else:
324 statement = text(query_select)
325 statement.bindparams(bindparam("copr_id", Integer))
326 result = db.engine.execute(statement, {"copr_id": copr.id})
327
328 return result
329
330 @classmethod
333
334 @classmethod
342
343 @classmethod
359
360 @classmethod
379
380 @classmethod
383
384 @classmethod
387
388 @classmethod
415
416 @classmethod
432
433 @classmethod
434 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
435 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
436 chroot_names=None, **build_options):
437 """
438 :type user: models.User
439 :type copr: models.Copr
440
441 :type chroot_names: List[str]
442
443 :rtype: models.Build
444 """
445 source_type = helpers.BuildSourceEnum("scm")
446 source_json = json.dumps({"type": scm_type,
447 "clone_url": clone_url,
448 "committish": committish,
449 "subdirectory": subdirectory,
450 "spec": spec,
451 "srpm_build_method": srpm_build_method})
452 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
453
454 @classmethod
455 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
456 chroot_names=None, **build_options):
473
474 @classmethod
487
488 @classmethod
489 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
490 chroot_names=None, **build_options):
491 """
492 :type user: models.User
493 :type copr: models.Copr
494 :param f_uploader(file_path): function which stores data at the given `file_path`
495 :return:
496 """
497 tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
498 tmp_name = os.path.basename(tmp)
499 filename = secure_filename(orig_filename)
500 file_path = os.path.join(tmp, filename)
501 f_uploader(file_path)
502
503
504 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
505 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
506 tmp_dir=tmp_name,
507 filename=filename)
508
509
510 source_type = helpers.BuildSourceEnum("upload")
511 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
512 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
513
514 try:
515 build = cls.create_new(user, copr, source_type, source_json,
516 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
517 except Exception:
518 shutil.rmtree(tmp)
519 raise
520
521 return build
522
523 @classmethod
524 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
525 git_hashes=None, skip_import=False, background=False, batch=None,
526 srpm_url=None, **build_options):
527 """
528 :type user: models.User
529 :type copr: models.Copr
530 :type chroot_names: List[str]
531 :type source_type: int value from helpers.BuildSourceEnum
532 :type source_json: str in json format
533 :type pkgs: str
534 :type git_hashes: dict
535 :type skip_import: bool
536 :type background: bool
537 :type batch: models.Batch
538 :rtype: models.Build
539 """
540 if chroot_names is None:
541 chroots = [c for c in copr.active_chroots]
542 else:
543 chroots = []
544 for chroot in copr.active_chroots:
545 if chroot.name in chroot_names:
546 chroots.append(chroot)
547
548 build = cls.add(
549 user=user,
550 pkgs=pkgs,
551 copr=copr,
552 chroots=chroots,
553 source_type=source_type,
554 source_json=source_json,
555 enable_net=build_options.get("enable_net", copr.build_enable_net),
556 background=background,
557 git_hashes=git_hashes,
558 skip_import=skip_import,
559 batch=batch,
560 srpm_url=srpm_url,
561 )
562
563 if user.proven:
564 if "timeout" in build_options:
565 build.timeout = build_options["timeout"]
566
567 return build
568
569 @classmethod
570 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
571 repos=None, chroots=None, timeout=None, enable_net=True,
572 git_hashes=None, skip_import=False, background=False, batch=None,
573 srpm_url=None):
574 if chroots is None:
575 chroots = []
576
577 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
578 copr, "Can't build while there is an operation in progress: {action}")
579 users_logic.UsersLogic.raise_if_cant_build_in_copr(
580 user, copr,
581 "You don't have permissions to build in this copr.")
582
583 if not repos:
584 repos = copr.repos
585
586
587 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
588 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
589 "with bad characters. Forgot to split?")
590
591
592 if not source_type or not source_json:
593 source_type = helpers.BuildSourceEnum("link")
594 source_json = json.dumps({"url":pkgs})
595
596 build = models.Build(
597 user=user,
598 pkgs=pkgs,
599 copr=copr,
600 repos=repos,
601 source_type=source_type,
602 source_json=source_json,
603 submitted_on=int(time.time()),
604 enable_net=bool(enable_net),
605 is_background=bool(background),
606 batch=batch,
607 srpm_url=srpm_url,
608 priority=(cls.get_task_lowest_priority(bool(background))+1)%MAX_PRIO
609 )
610
611 if timeout:
612 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
613
614 db.session.add(build)
615
616
617
618 if not chroots:
619 chroots = copr.active_chroots
620
621 if skip_import:
622 status = StatusEnum("pending")
623 priority = (cls.get_task_lowest_priority(bool(background))+1)%MAX_PRIO
624 else:
625 status = StatusEnum("importing")
626 priority = 0
627
628 for chroot in chroots:
629 git_hash = None
630 if git_hashes:
631 git_hash = git_hashes.get(chroot.name)
632 buildchroot = models.BuildChroot(
633 build=build,
634 status=status,
635 mock_chroot=chroot,
636 git_hash=git_hash,
637 priority=priority
638 )
639 db.session.add(buildchroot)
640
641 return build
642
643 @classmethod
645 source_dict = package.source_json_dict
646 source_dict.update(source_dict_update)
647 source_json = json.dumps(source_dict)
648
649 build = models.Build(
650 user=None,
651 pkgs=None,
652 package_id=package.id,
653 copr=package.copr,
654 repos=package.copr.repos,
655 source_type=package.source_type,
656 source_json=source_json,
657 submitted_on=int(time.time()),
658 enable_net=package.copr.build_enable_net,
659 timeout=DEFAULT_BUILD_TIMEOUT
660 )
661
662 db.session.add(build)
663
664 chroots = package.copr.active_chroots
665
666 status = helpers.StatusEnum("importing")
667
668 for chroot in chroots:
669 buildchroot = models.BuildChroot(
670 build=build,
671 status=status,
672 mock_chroot=chroot,
673 git_hash=None
674 )
675
676 db.session.add(buildchroot)
677
678 return build
679
680
681 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
682
683 @classmethod
695
696
697 @classmethod
711
712
713 @classmethod
715 """
716 :param build:
717 :param upd_dict:
718 example:
719 {
720 "builds":[
721 {
722 "id": 1,
723 "copr_id": 2,
724 "started_on": 139086644000
725 },
726 {
727 "id": 2,
728 "copr_id": 1,
729 "status": 0,
730 "chroot": "fedora-18-x86_64",
731 "results": "http://server/results/foo/bar/",
732 "ended_on": 139086644000
733 }]
734 }
735 """
736 log.info("Updating build: {} by: {}".format(build.id, upd_dict))
737 if "chroot" in upd_dict:
738 if upd_dict["chroot"] == "srpm-builds":
739 if upd_dict.get("status") == StatusEnum("failed") and not build.canceled:
740 build.fail_type = helpers.FailTypeEnum("srpm_build_error")
741 for ch in build.build_chroots:
742 ch.status = helpers.StatusEnum("failed")
743 ch.ended_on = upd_dict.get("ended_on") or time.time()
744 db.session.add(ch)
745
746
747 for build_chroot in build.build_chroots:
748 if build_chroot.name == upd_dict["chroot"]:
749
750 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
751 build_chroot.status = upd_dict["status"]
752
753 if upd_dict.get("status") in BuildsLogic.terminal_states:
754 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
755
756 if upd_dict.get("status") == StatusEnum("starting"):
757 build_chroot.started_on = upd_dict.get("started_on") or time.time()
758
759 db.session.add(build_chroot)
760
761 for attr in ["results", "built_packages", "srpm_url"]:
762 value = upd_dict.get(attr, None)
763 if value:
764 setattr(build, attr, value)
765
766 db.session.add(build)
767
768 @classmethod
788
789 @classmethod
790 - def delete_build(cls, user, build, send_delete_action=True):
811
812 @classmethod
822
823 @classmethod
842
843 @classmethod
851
852 @classmethod
855
858 @classmethod
867
868 @classmethod
869 - def defer(cls, build_id, name):
876
877 @classmethod
888
889 @classmethod
892
893 @classmethod
896
897 @classmethod
900
901 @classmethod
904
905 @classmethod
908
911 @classmethod
913 query = """
914 SELECT
915 package.id as package_id,
916 package.name AS package_name,
917 build.id AS build_id,
918 build_chroot.status AS build_chroot_status,
919 build.pkg_version AS build_pkg_version,
920 mock_chroot.id AS mock_chroot_id,
921 mock_chroot.os_release AS mock_chroot_os_release,
922 mock_chroot.os_version AS mock_chroot_os_version,
923 mock_chroot.arch AS mock_chroot_arch
924 FROM package
925 JOIN (SELECT
926 MAX(build.id) AS max_build_id_for_chroot,
927 build.package_id AS package_id,
928 build_chroot.mock_chroot_id AS mock_chroot_id
929 FROM build
930 JOIN build_chroot
931 ON build.id = build_chroot.build_id
932 WHERE build.copr_id = {copr_id}
933 AND build_chroot.status != 2
934 GROUP BY build.package_id,
935 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
936 ON package.id = max_build_ids_for_a_chroot.package_id
937 JOIN build
938 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
939 JOIN build_chroot
940 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
941 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
942 JOIN mock_chroot
943 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
944 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
945 """.format(copr_id=copr.id)
946 rows = db.session.execute(query)
947 return rows
948