1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 from sqlalchemy.sql import text
10 from sqlalchemy import or_
11 from sqlalchemy import and_
12 from sqlalchemy.orm import joinedload
13 from sqlalchemy.orm.exc import NoResultFound
14 from sqlalchemy.sql import false,true
15 from werkzeug.utils import secure_filename
16 from sqlalchemy import desc,asc, bindparam, Integer
17 from collections import defaultdict
18
19 from coprs import app
20 from coprs import db
21 from coprs import exceptions
22 from coprs import models
23 from coprs import helpers
24 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
25 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException
26 from coprs.helpers import StatusEnum
27
28 from coprs.logic import coprs_logic
29 from coprs.logic import users_logic
30 from coprs.logic.actions_logic import ActionsLogic
31 from coprs.models import BuildChroot,Build,Package,MockChroot
32 from .coprs_logic import MockChrootsLogic
33
34 log = app.logger
119
120 @classmethod
128
129 @classmethod
147
148 @classmethod
157
158 @classmethod
161
162 @classmethod
165
166 @classmethod
171
172 @classmethod
179
180
181 @classmethod
183 if db.engine.url.drivername == "sqlite":
184 return
185
186 status_to_order = """
187 CREATE OR REPLACE FUNCTION status_to_order (x integer)
188 RETURNS integer AS $$ BEGIN
189 RETURN CASE WHEN x = 0 THEN 0
190 WHEN x = 3 THEN 1
191 WHEN x = 6 THEN 2
192 WHEN x = 7 THEN 3
193 WHEN x = 4 THEN 4
194 WHEN x = 1 THEN 5
195 WHEN x = 5 THEN 6
196 ELSE 1000
197 END; END;
198 $$ LANGUAGE plpgsql;
199 """
200
201 order_to_status = """
202 CREATE OR REPLACE FUNCTION order_to_status (x integer)
203 RETURNS integer AS $$ BEGIN
204 RETURN CASE WHEN x = 0 THEN 0
205 WHEN x = 1 THEN 3
206 WHEN x = 2 THEN 6
207 WHEN x = 3 THEN 7
208 WHEN x = 4 THEN 4
209 WHEN x = 5 THEN 1
210 WHEN x = 6 THEN 5
211 ELSE 1000
212 END; END;
213 $$ LANGUAGE plpgsql;
214 """
215
216 db.engine.connect()
217 db.engine.execute(status_to_order)
218 db.engine.execute(order_to_status)
219
220 @classmethod
222 query_select = """
223 SELECT build.id, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
224 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
225 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name
226 FROM build
227 LEFT OUTER JOIN package
228 ON build.package_id = package.id
229 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
230 ON statuses.build_id=build.id
231 LEFT OUTER JOIN copr
232 ON copr.id = build.copr_id
233 LEFT OUTER JOIN "user"
234 ON copr.user_id = "user".id
235 LEFT OUTER JOIN "group"
236 ON copr.group_id = "group".id
237 WHERE build.copr_id = :copr_id
238 GROUP BY
239 build.id;
240 """
241
242 if db.engine.url.drivername == "sqlite":
243 def sqlite_status_to_order(x):
244 if x == 3:
245 return 1
246 elif x == 6:
247 return 2
248 elif x == 7:
249 return 3
250 elif x == 4:
251 return 4
252 elif x == 0:
253 return 5
254 elif x == 1:
255 return 6
256 elif x == 5:
257 return 7
258 elif x == 8:
259 return 8
260 return 1000
261
262 def sqlite_order_to_status(x):
263 if x == 1:
264 return 3
265 elif x == 2:
266 return 6
267 elif x == 3:
268 return 7
269 elif x == 4:
270 return 4
271 elif x == 5:
272 return 0
273 elif x == 6:
274 return 1
275 elif x == 7:
276 return 5
277 elif x == 8:
278 return 8
279 return 1000
280
281 conn = db.engine.connect()
282 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
283 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
284 statement = text(query_select)
285 statement.bindparams(bindparam("copr_id", Integer))
286 result = conn.execute(statement, {"copr_id": copr.id})
287 else:
288 statement = text(query_select)
289 statement.bindparams(bindparam("copr_id", Integer))
290 result = db.engine.execute(statement, {"copr_id": copr.id})
291
292 return result
293
294 @classmethod
297
298 @classmethod
306
307 @classmethod
323
324 @classmethod
343
344 @classmethod
347
348 @classmethod
351
352 @classmethod
379
380 @classmethod
396
397 @classmethod
398 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
399 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
400 chroot_names=None, **build_options):
401 """
402 :type user: models.User
403 :type copr: models.Copr
404
405 :type chroot_names: List[str]
406
407 :rtype: models.Build
408 """
409 source_type = helpers.BuildSourceEnum("scm")
410 source_json = json.dumps({"type": scm_type,
411 "clone_url": clone_url,
412 "committish": committish,
413 "subdirectory": subdirectory,
414 "spec": spec,
415 "srpm_build_method": srpm_build_method})
416 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
417
418 @classmethod
419 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
420 chroot_names=None, **build_options):
437
438 @classmethod
451
452 @classmethod
453 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
454 chroot_names=None, **build_options):
455 """
456 :type user: models.User
457 :type copr: models.Copr
458 :param f_uploader(file_path): function which stores data at the given `file_path`
459 :return:
460 """
461 tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"])
462 tmp_name = os.path.basename(tmp)
463 filename = secure_filename(orig_filename)
464 file_path = os.path.join(tmp, filename)
465 f_uploader(file_path)
466
467
468 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
469 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
470 tmp_dir=tmp_name,
471 filename=filename)
472
473
474 source_type = helpers.BuildSourceEnum("upload")
475 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
476 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
477
478 try:
479 build = cls.create_new(user, copr, source_type, source_json,
480 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
481 except Exception:
482 shutil.rmtree(tmp)
483 raise
484
485 return build
486
487 @classmethod
488 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
489 git_hashes=None, skip_import=False, background=False, batch=None,
490 srpm_url=None, **build_options):
491 """
492 :type user: models.User
493 :type copr: models.Copr
494 :type chroot_names: List[str]
495 :type source_type: int value from helpers.BuildSourceEnum
496 :type source_json: str in json format
497 :type pkgs: str
498 :type git_hashes: dict
499 :type skip_import: bool
500 :type background: bool
501 :type batch: models.Batch
502 :rtype: models.Build
503 """
504 if chroot_names is None:
505 chroots = [c for c in copr.active_chroots]
506 else:
507 chroots = []
508 for chroot in copr.active_chroots:
509 if chroot.name in chroot_names:
510 chroots.append(chroot)
511
512 build = cls.add(
513 user=user,
514 pkgs=pkgs,
515 copr=copr,
516 chroots=chroots,
517 source_type=source_type,
518 source_json=source_json,
519 enable_net=build_options.get("enable_net", copr.build_enable_net),
520 background=background,
521 git_hashes=git_hashes,
522 skip_import=skip_import,
523 batch=batch,
524 srpm_url=srpm_url,
525 )
526
527 if user.proven:
528 if "timeout" in build_options:
529 build.timeout = build_options["timeout"]
530
531 return build
532
533 @classmethod
534 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
535 repos=None, chroots=None, timeout=None, enable_net=True,
536 git_hashes=None, skip_import=False, background=False, batch=None,
537 srpm_url=None):
538 if chroots is None:
539 chroots = []
540
541 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
542 copr, "Can't build while there is an operation in progress: {action}")
543 users_logic.UsersLogic.raise_if_cant_build_in_copr(
544 user, copr,
545 "You don't have permissions to build in this copr.")
546
547 if not repos:
548 repos = copr.repos
549
550
551 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
552 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
553 "with bad characters. Forgot to split?")
554
555
556 if not source_type or not source_json:
557 source_type = helpers.BuildSourceEnum("link")
558 source_json = json.dumps({"url":pkgs})
559
560 build = models.Build(
561 user=user,
562 pkgs=pkgs,
563 copr=copr,
564 repos=repos,
565 source_type=source_type,
566 source_json=source_json,
567 submitted_on=int(time.time()),
568 enable_net=bool(enable_net),
569 is_background=bool(background),
570 batch=batch,
571 srpm_url=srpm_url,
572 )
573
574 if timeout:
575 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
576
577 db.session.add(build)
578
579
580
581 if not chroots:
582 chroots = copr.active_chroots
583
584 status = helpers.StatusEnum("importing")
585
586 if skip_import:
587 status = StatusEnum("pending")
588
589 for chroot in chroots:
590 git_hash = None
591 if git_hashes:
592 git_hash = git_hashes.get(chroot.name)
593 buildchroot = models.BuildChroot(
594 build=build,
595 status=status,
596 mock_chroot=chroot,
597 git_hash=git_hash)
598
599 db.session.add(buildchroot)
600
601 return build
602
603 @classmethod
605 source_dict = package.source_json_dict
606 source_dict.update(source_dict_update)
607 source_json = json.dumps(source_dict)
608
609 build = models.Build(
610 user=None,
611 pkgs=None,
612 package_id=package.id,
613 copr=package.copr,
614 repos=package.copr.repos,
615 source_type=package.source_type,
616 source_json=source_json,
617 submitted_on=int(time.time()),
618 enable_net=package.copr.build_enable_net,
619 timeout=DEFAULT_BUILD_TIMEOUT
620 )
621
622 db.session.add(build)
623
624 chroots = package.copr.active_chroots
625
626 status = helpers.StatusEnum("importing")
627
628 for chroot in chroots:
629 buildchroot = models.BuildChroot(
630 build=build,
631 status=status,
632 mock_chroot=chroot,
633 git_hash=None
634 )
635
636 db.session.add(buildchroot)
637
638 return build
639
640
641 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
642
643 @classmethod
655
656
657 @classmethod
671
672
673 @classmethod
675 """
676 :param build:
677 :param upd_dict:
678 example:
679 {
680 "builds":[
681 {
682 "id": 1,
683 "copr_id": 2,
684 "started_on": 139086644000
685 },
686 {
687 "id": 2,
688 "copr_id": 1,
689 "status": 0,
690 "chroot": "fedora-18-x86_64",
691 "results": "http://server/results/foo/bar/",
692 "ended_on": 139086644000
693 }]
694 }
695 """
696 log.info("Updating build: {} by: {}".format(build.id, upd_dict))
697 if "chroot" in upd_dict:
698 if upd_dict["chroot"] == "srpm-builds":
699 if upd_dict.get("status") == StatusEnum("failed") and not build.canceled:
700 build.fail_type = helpers.FailTypeEnum("srpm_build_error")
701 for ch in build.build_chroots:
702 ch.status = helpers.StatusEnum("failed")
703 ch.ended_on = upd_dict.get("ended_on") or time.time()
704 db.session.add(ch)
705
706
707 for build_chroot in build.build_chroots:
708 if build_chroot.name == upd_dict["chroot"]:
709
710 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
711 build_chroot.status = upd_dict["status"]
712
713 if upd_dict.get("status") in BuildsLogic.terminal_states:
714 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
715
716 if upd_dict.get("status") == StatusEnum("starting"):
717 build_chroot.started_on = upd_dict.get("started_on") or time.time()
718
719 if "last_deferred" in upd_dict:
720 build_chroot.last_deferred = upd_dict["last_deferred"]
721
722 db.session.add(build_chroot)
723
724 for attr in ["results", "built_packages", "srpm_url"]:
725 value = upd_dict.get(attr, None)
726 if value:
727 setattr(build, attr, value)
728
729 db.session.add(build)
730
731 @classmethod
751
752 @classmethod
753 - def delete_build(cls, user, build, send_delete_action=True):
774
775 @classmethod
785
786 @classmethod
805
806 @classmethod
814
815 @classmethod
818
821 @classmethod
830
831 @classmethod
842
843 @classmethod
846
847 @classmethod
850
851 @classmethod
854
855 @classmethod
858
859 @classmethod
862
865 @classmethod
867 query = """
868 SELECT
869 package.id as package_id,
870 package.name AS package_name,
871 build.id AS build_id,
872 build_chroot.status AS build_chroot_status,
873 build.pkg_version AS build_pkg_version,
874 mock_chroot.id AS mock_chroot_id,
875 mock_chroot.os_release AS mock_chroot_os_release,
876 mock_chroot.os_version AS mock_chroot_os_version,
877 mock_chroot.arch AS mock_chroot_arch
878 FROM package
879 JOIN (SELECT
880 MAX(build.id) AS max_build_id_for_chroot,
881 build.package_id AS package_id,
882 build_chroot.mock_chroot_id AS mock_chroot_id
883 FROM build
884 JOIN build_chroot
885 ON build.id = build_chroot.build_id
886 WHERE build.copr_id = {copr_id}
887 AND build_chroot.status != 2
888 GROUP BY build.package_id,
889 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
890 ON package.id = max_build_ids_for_a_chroot.package_id
891 JOIN build
892 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
893 JOIN build_chroot
894 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
895 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
896 JOIN mock_chroot
897 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
898 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
899 """.format(copr_id=copr.id)
900 rows = db.session.execute(query)
901 return rows
902