1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 import requests
10
11 from flask import request
12 from sqlalchemy.sql import text
13 from sqlalchemy import or_
14 from sqlalchemy import and_
15 from sqlalchemy import func
16 from sqlalchemy.orm import joinedload
17 from sqlalchemy.orm.exc import NoResultFound
18 from sqlalchemy.sql import false,true
19 from werkzeug.utils import secure_filename
20 from sqlalchemy import desc, asc, bindparam, Integer, String
21 from collections import defaultdict
22
23 from coprs import app
24 from coprs import db
25 from coprs import exceptions
26 from coprs import models
27 from coprs import helpers
28 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
29 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException
30 from coprs.helpers import StatusEnum
31
32 from coprs.logic import coprs_logic
33 from coprs.logic import users_logic
34 from coprs.logic.actions_logic import ActionsLogic
35 from coprs.models import BuildChroot,Build,Package,MockChroot
36 from .coprs_logic import MockChrootsLogic
37
38 log = app.logger
42 @classmethod
43 - def get(cls, build_id):
45
46 @classmethod
57
58 @classmethod
69
70 @classmethod
89
90 @classmethod
98
99 @classmethod
101 end = int(time.time())
102 start = end - 86399
103 step = 3600
104 tasks = cls.get_running_tasks_by_time(start, end)
105 steps = int(round((end - start) / step + 0.5))
106 current_step = 0
107
108 data = [[0] * (steps + 1)]
109 data[0][0] = ''
110 for t in tasks:
111 task = t.to_dict()
112 while task['started_on'] > start + step * (current_step + 1):
113 current_step += 1
114 data[0][current_step + 1] += 1
115 return data
116
117 @classmethod
138
139 @classmethod
141 start = start - (start % step)
142 end = end - (end % step)
143 steps = int((end - start) / step + 0.5)
144 data = [['pending'], ['running'], ['avg running'], ['time']]
145
146 result = models.BuildsStatistics.query\
147 .filter(models.BuildsStatistics.stat_type == type)\
148 .filter(models.BuildsStatistics.time >= start)\
149 .filter(models.BuildsStatistics.time <= end)\
150 .order_by(models.BuildsStatistics.time)
151
152 for row in result:
153 data[0].append(row.pending)
154 data[1].append(row.running)
155
156 for i in range(len(data[0]) - 1, steps):
157 step_start = start + i * step
158 step_end = step_start + step
159
160 query_pending = text("""
161 SELECT COUNT(*) as pending
162 FROM build_chroot JOIN build on build.id = build_chroot.build_id
163 WHERE
164 build.submitted_on < :end
165 AND (
166 build_chroot.started_on > :start
167 OR (build_chroot.started_on is NULL AND build_chroot.status = :status)
168 -- for currently pending builds we need to filter on status=pending because there might be
169 -- failed builds that have started_on=NULL
170 )
171 AND NOT build.canceled
172 """)
173
174 query_running = text("""
175 SELECT COUNT(*) as running
176 FROM build_chroot
177 WHERE
178 started_on < :end
179 AND (ended_on > :start OR (ended_on is NULL AND status = :status))
180 -- for currently running builds we need to filter on status=running because there might be failed
181 -- builds that have ended_on=NULL
182 """)
183
184 res_pending = db.engine.execute(query_pending, start=step_start, end=step_end,
185 status=helpers.StatusEnum('pending'))
186 res_running = db.engine.execute(query_running, start=step_start, end=step_end,
187 status=helpers.StatusEnum('running'))
188
189 pending = res_pending.first().pending
190 running = res_running.first().running
191 data[0].append(pending)
192 data[1].append(running)
193
194 statistic = models.BuildsStatistics(
195 time = step_start,
196 stat_type = type,
197 running = running,
198 pending = pending
199 )
200 db.session.merge(statistic)
201 db.session.commit()
202
203 running_total = 0
204 for i in range(1, steps + 1):
205 running_total += data[1][i]
206
207 data[2].extend([running_total * 1.0 / steps] * (len(data[0]) - 1))
208
209 for i in range(start, end, step):
210 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i)))
211
212 return data
213
214 @classmethod
226
227 @classmethod
236
237 @classmethod
253
254 @classmethod
263
264 @classmethod
267
268 @classmethod
271
272 @classmethod
277
278 @classmethod
285
286 @classmethod
288 if db.engine.url.drivername == "sqlite":
289 return
290
291 status_to_order = """
292 CREATE OR REPLACE FUNCTION status_to_order (x integer)
293 RETURNS integer AS $$ BEGIN
294 RETURN CASE WHEN x = 3 THEN 1
295 WHEN x = 6 THEN 2
296 WHEN x = 7 THEN 3
297 WHEN x = 4 THEN 4
298 WHEN x = 0 THEN 5
299 WHEN x = 1 THEN 6
300 WHEN x = 5 THEN 7
301 WHEN x = 2 THEN 8
302 WHEN x = 8 THEN 9
303 WHEN x = 9 THEN 10
304 ELSE x
305 END; END;
306 $$ LANGUAGE plpgsql;
307 """
308
309 order_to_status = """
310 CREATE OR REPLACE FUNCTION order_to_status (x integer)
311 RETURNS integer AS $$ BEGIN
312 RETURN CASE WHEN x = 1 THEN 3
313 WHEN x = 2 THEN 6
314 WHEN x = 3 THEN 7
315 WHEN x = 4 THEN 4
316 WHEN x = 5 THEN 0
317 WHEN x = 6 THEN 1
318 WHEN x = 7 THEN 5
319 WHEN x = 8 THEN 2
320 WHEN x = 9 THEN 8
321 WHEN x = 10 THEN 9
322 ELSE x
323 END; END;
324 $$ LANGUAGE plpgsql;
325 """
326
327 db.engine.connect()
328 db.engine.execute(status_to_order)
329 db.engine.execute(order_to_status)
330
331 @classmethod
333 query_select = """
334 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
335 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
336 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id
337 FROM build
338 LEFT OUTER JOIN package
339 ON build.package_id = package.id
340 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
341 ON statuses.build_id=build.id
342 LEFT OUTER JOIN copr
343 ON copr.id = build.copr_id
344 LEFT OUTER JOIN copr_dir
345 ON build.copr_dir_id = copr_dir.id
346 LEFT OUTER JOIN "user"
347 ON copr.user_id = "user".id
348 LEFT OUTER JOIN "group"
349 ON copr.group_id = "group".id
350 WHERE build.copr_id = :copr_id
351 AND (:dirname = '' OR :dirname = copr_dir.name)
352 GROUP BY
353 build.id;
354 """
355
356 if db.engine.url.drivername == "sqlite":
357 def sqlite_status_to_order(x):
358 if x == 3:
359 return 1
360 elif x == 6:
361 return 2
362 elif x == 7:
363 return 3
364 elif x == 4:
365 return 4
366 elif x == 0:
367 return 5
368 elif x == 1:
369 return 6
370 elif x == 5:
371 return 7
372 elif x == 2:
373 return 8
374 elif x == 8:
375 return 9
376 elif x == 9:
377 return 10
378 return 1000
379
380 def sqlite_order_to_status(x):
381 if x == 1:
382 return 3
383 elif x == 2:
384 return 6
385 elif x == 3:
386 return 7
387 elif x == 4:
388 return 4
389 elif x == 5:
390 return 0
391 elif x == 6:
392 return 1
393 elif x == 7:
394 return 5
395 elif x == 8:
396 return 2
397 elif x == 9:
398 return 8
399 elif x == 10:
400 return 9
401 return 1000
402
403 conn = db.engine.connect()
404 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
405 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
406 statement = text(query_select)
407 statement.bindparams(bindparam("copr_id", Integer))
408 statement.bindparams(bindparam("dirname", String))
409 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname})
410 else:
411 statement = text(query_select)
412 statement.bindparams(bindparam("copr_id", Integer))
413 statement.bindparams(bindparam("dirname", String))
414 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname})
415
416 return result
417
418 @classmethod
421
422 @classmethod
430
431 @classmethod
434
435 @classmethod
438
439 @classmethod
459
460 @classmethod
476
477 @classmethod
478 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
479 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
480 chroot_names=None, **build_options):
481 """
482 :type user: models.User
483 :type copr: models.Copr
484
485 :type chroot_names: List[str]
486
487 :rtype: models.Build
488 """
489 source_type = helpers.BuildSourceEnum("scm")
490 source_json = json.dumps({"type": scm_type,
491 "clone_url": clone_url,
492 "committish": committish,
493 "subdirectory": subdirectory,
494 "spec": spec,
495 "srpm_build_method": srpm_build_method})
496 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
497
498 @classmethod
499 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
500 chroot_names=None, **build_options):
517
518 @classmethod
531
532 @classmethod
533 - def create_new_from_custom(cls, user, copr,
534 script, script_chroot=None, script_builddeps=None,
535 script_resultdir=None, chroot_names=None, **kwargs):
536 """
537 :type user: models.User
538 :type copr: models.Copr
539 :type script: str
540 :type script_chroot: str
541 :type script_builddeps: str
542 :type script_resultdir: str
543 :type chroot_names: List[str]
544 :rtype: models.Build
545 """
546 source_type = helpers.BuildSourceEnum("custom")
547 source_dict = {
548 'script': script,
549 'chroot': script_chroot,
550 'builddeps': script_builddeps,
551 'resultdir': script_resultdir,
552 }
553
554 return cls.create_new(user, copr, source_type, json.dumps(source_dict),
555 chroot_names, **kwargs)
556
557 @classmethod
558 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
559 chroot_names=None, **build_options):
560 """
561 :type user: models.User
562 :type copr: models.Copr
563 :param f_uploader(file_path): function which stores data at the given `file_path`
564 :return:
565 """
566 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
567 tmp_name = os.path.basename(tmp)
568 filename = secure_filename(orig_filename)
569 file_path = os.path.join(tmp, filename)
570 f_uploader(file_path)
571
572
573 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
574 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
575 tmp_dir=tmp_name,
576 filename=filename)
577
578
579 source_type = helpers.BuildSourceEnum("upload")
580 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
581 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
582
583 try:
584 build = cls.create_new(user, copr, source_type, source_json,
585 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
586 except Exception:
587 shutil.rmtree(tmp)
588 raise
589
590 return build
591
592 @classmethod
593 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
594 git_hashes=None, skip_import=False, background=False, batch=None,
595 srpm_url=None, **build_options):
596 """
597 :type user: models.User
598 :type copr: models.Copr
599 :type chroot_names: List[str]
600 :type source_type: int value from helpers.BuildSourceEnum
601 :type source_json: str in json format
602 :type pkgs: str
603 :type git_hashes: dict
604 :type skip_import: bool
605 :type background: bool
606 :type batch: models.Batch
607 :rtype: models.Build
608 """
609 if chroot_names is None:
610 chroots = [c for c in copr.active_chroots]
611 else:
612 chroots = []
613 for chroot in copr.active_chroots:
614 if chroot.name in chroot_names:
615 chroots.append(chroot)
616
617 build = cls.add(
618 user=user,
619 pkgs=pkgs,
620 copr=copr,
621 chroots=chroots,
622 source_type=source_type,
623 source_json=source_json,
624 enable_net=build_options.get("enable_net", copr.build_enable_net),
625 background=background,
626 git_hashes=git_hashes,
627 skip_import=skip_import,
628 batch=batch,
629 srpm_url=srpm_url,
630 )
631
632 if user.proven:
633 if "timeout" in build_options:
634 build.timeout = build_options["timeout"]
635
636 return build
637
638 @classmethod
639 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
640 repos=None, chroots=None, timeout=None, enable_net=True,
641 git_hashes=None, skip_import=False, background=False, batch=None,
642 srpm_url=None):
643
644 if chroots is None:
645 chroots = []
646
647 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
648 copr, "Can't build while there is an operation in progress: {action}")
649 users_logic.UsersLogic.raise_if_cant_build_in_copr(
650 user, copr,
651 "You don't have permissions to build in this copr.")
652
653 if not repos:
654 repos = copr.repos
655
656
657 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
658 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
659 "with bad characters. Forgot to split?")
660
661
662 if not source_type or not source_json:
663 source_type = helpers.BuildSourceEnum("link")
664 source_json = json.dumps({"url":pkgs})
665
666 if skip_import and srpm_url:
667 chroot_status = StatusEnum("pending")
668 source_status = StatusEnum("succeeded")
669 elif srpm_url:
670 chroot_status = StatusEnum("waiting")
671 source_status = StatusEnum("importing")
672 else:
673 chroot_status = StatusEnum("waiting")
674 source_status = StatusEnum("pending")
675
676 build = models.Build(
677 user=user,
678 pkgs=pkgs,
679 copr=copr,
680 repos=repos,
681 source_type=source_type,
682 source_json=source_json,
683 source_status=source_status,
684 submitted_on=int(time.time()),
685 enable_net=bool(enable_net),
686 is_background=bool(background),
687 batch=batch,
688 srpm_url=srpm_url,
689 )
690
691 if timeout:
692 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
693
694 db.session.add(build)
695
696
697
698 if not chroots:
699 chroots = copr.active_chroots
700
701 for chroot in chroots:
702 git_hash = None
703 if git_hashes:
704 git_hash = git_hashes.get(chroot.name)
705 buildchroot = models.BuildChroot(
706 build=build,
707 status=chroot_status,
708 mock_chroot=chroot,
709 git_hash=git_hash,
710 )
711 db.session.add(buildchroot)
712
713 return build
714
715 @classmethod
716 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None,
717 scm_object_type=None, scm_object_id=None, scm_object_url=None):
718
719 source_dict = package.source_json_dict
720 source_dict.update(source_dict_update)
721 source_json = json.dumps(source_dict)
722
723 if not copr_dir:
724 copr_dir = package.copr.main_dir
725
726 build = models.Build(
727 user=None,
728 pkgs=None,
729 package=package,
730 copr=package.copr,
731 repos=package.copr.repos,
732 source_status=helpers.StatusEnum("pending"),
733 source_type=package.source_type,
734 source_json=source_json,
735 submitted_on=int(time.time()),
736 enable_net=package.copr.build_enable_net,
737 timeout=DEFAULT_BUILD_TIMEOUT,
738 copr_dir=copr_dir,
739 update_callback=update_callback,
740 scm_object_type=scm_object_type,
741 scm_object_id=scm_object_id,
742 scm_object_url=scm_object_url,
743 )
744 db.session.add(build)
745
746 chroots = package.copr.active_chroots
747 status = helpers.StatusEnum("waiting")
748 for chroot in chroots:
749 buildchroot = models.BuildChroot(
750 build=build,
751 status=status,
752 mock_chroot=chroot,
753 git_hash=None
754 )
755 db.session.add(buildchroot)
756
757 cls.process_update_callback(build)
758 return build
759
760
761 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
762
763 @classmethod
775
776
777 @classmethod
779 """
780 Deletes the locally stored data for build purposes. This is typically
781 uploaded srpm file, uploaded spec file or webhook POST content.
782 """
783
784 data = json.loads(build.source_json)
785 if 'tmp' in data:
786 tmp = data["tmp"]
787 storage_path = app.config["STORAGE_DIR"]
788 try:
789 shutil.rmtree(os.path.join(storage_path, tmp))
790 except:
791 pass
792
793
794 @classmethod
796 """
797 :param build:
798 :param upd_dict:
799 example:
800 {
801 "builds":[
802 {
803 "id": 1,
804 "copr_id": 2,
805 "started_on": 1390866440
806 },
807 {
808 "id": 2,
809 "copr_id": 1,
810 "status": 0,
811 "chroot": "fedora-18-x86_64",
812 "result_dir": "baz",
813 "ended_on": 1390866440
814 }]
815 }
816 """
817 log.info("Updating build {} by: {}".format(build.id, upd_dict))
818
819
820 for attr in ["built_packages", "srpm_url"]:
821 value = upd_dict.get(attr, None)
822 if value:
823 setattr(build, attr, value)
824
825
826 if upd_dict.get("task_id") == build.task_id:
827 build.result_dir = upd_dict.get("result_dir", "")
828
829 if upd_dict.get("status") == StatusEnum("succeeded"):
830 new_status = StatusEnum("importing")
831 else:
832 new_status = upd_dict.get("status")
833
834 build.source_status = new_status
835 if new_status == StatusEnum("failed") or \
836 new_status == StatusEnum("skipped"):
837 for ch in build.build_chroots:
838 ch.status = new_status
839 ch.ended_on = upd_dict.get("ended_on") or time.time()
840 db.session.add(ch)
841
842 if new_status == StatusEnum("failed"):
843 build.fail_type = helpers.FailTypeEnum("srpm_build_error")
844
845 cls.process_update_callback(build)
846 db.session.add(build)
847 return
848
849 if "chroot" in upd_dict:
850
851 for build_chroot in build.build_chroots:
852 if build_chroot.name == upd_dict["chroot"]:
853 build_chroot.result_dir = upd_dict.get("result_dir", "")
854
855 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
856 build_chroot.status = upd_dict["status"]
857
858 if upd_dict.get("status") in BuildsLogic.terminal_states:
859 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
860
861 if upd_dict.get("status") == StatusEnum("starting"):
862 build_chroot.started_on = upd_dict.get("started_on") or time.time()
863
864 db.session.add(build_chroot)
865
866
867
868 if (build.module
869 and upd_dict.get("status") == StatusEnum("succeeded")
870 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)):
871 ActionsLogic.send_build_module(build.copr, build.module)
872
873 cls.process_update_callback(build)
874 db.session.add(build)
875
876 @classmethod
891
892 @classmethod
894 headers = {
895 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key'))
896 }
897
898 if build.srpm_url:
899 progress = 50
900 else:
901 progress = 10
902
903 state_table = {
904 'failed': ('failure', 0),
905 'succeeded': ('success', 100),
906 'canceled': ('canceled', 0),
907 'running': ('pending', progress),
908 'pending': ('pending', progress),
909 'skipped': ('error', 0),
910 'starting': ('pending', progress),
911 'importing': ('pending', progress),
912 'forked': ('error', 0),
913 'waiting': ('pending', progress),
914 'unknown': ('error', 0),
915 }
916
917 build_url = os.path.join(
918 app.config['PUBLIC_COPR_BASE_URL'],
919 'coprs', build.copr.full_name.replace('@', 'g/'),
920 'build', str(build.id)
921 )
922
923 data = {
924 'username': 'Copr build',
925 'comment': '#{}'.format(build.id),
926 'url': build_url,
927 'status': state_table[build.state][0],
928 'percent': state_table[build.state][1],
929 'uid': str(build.id),
930 }
931
932 log.info('Sending data to Pagure API: %s', pprint.pformat(data))
933 response = requests.post(api_url, data=data, headers=headers)
934 log.info('Pagure API response: %s', response.text)
935
936 @classmethod
959
960 @classmethod
961 - def delete_build(cls, user, build, send_delete_action=True):
982
983 @classmethod
994
995 @classmethod
1014
1015 @classmethod
1023
1024 @classmethod
1027
1028 @classmethod
1031
1034 @classmethod
1043
1044 @classmethod
1055
1056 @classmethod
1059
1060 @classmethod
1063
1064 @classmethod
1067
1068 @classmethod
1071
1072 @classmethod
1075
1078 @classmethod
1080 query = """
1081 SELECT
1082 package.id as package_id,
1083 package.name AS package_name,
1084 build.id AS build_id,
1085 build_chroot.status AS build_chroot_status,
1086 build.pkg_version AS build_pkg_version,
1087 mock_chroot.id AS mock_chroot_id,
1088 mock_chroot.os_release AS mock_chroot_os_release,
1089 mock_chroot.os_version AS mock_chroot_os_version,
1090 mock_chroot.arch AS mock_chroot_arch
1091 FROM package
1092 JOIN (SELECT
1093 MAX(build.id) AS max_build_id_for_chroot,
1094 build.package_id AS package_id,
1095 build_chroot.mock_chroot_id AS mock_chroot_id
1096 FROM build
1097 JOIN build_chroot
1098 ON build.id = build_chroot.build_id
1099 WHERE build.copr_id = {copr_id}
1100 AND build_chroot.status != 2
1101 GROUP BY build.package_id,
1102 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
1103 ON package.id = max_build_ids_for_a_chroot.package_id
1104 JOIN build
1105 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1106 JOIN build_chroot
1107 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
1108 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
1109 JOIN mock_chroot
1110 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
1111 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
1112 """.format(copr_id=copr.id)
1113 rows = db.session.execute(query)
1114 return rows
1115