Package coprs :: Package views :: Package backend_ns :: Module backend_general
[hide private]
[frames] | no frames]

Source Code for Module coprs.views.backend_ns.backend_general

  1  import flask 
  2  import time 
  3  import sqlalchemy 
  4   
  5  from coprs import db, app 
  6  from coprs import helpers 
  7  from coprs import models 
  8  from coprs import exceptions 
  9  from coprs.helpers import StatusEnum 
 10  from coprs.logic import actions_logic 
 11  from coprs.logic.builds_logic import BuildsLogic, BuildChrootsLogic 
 12  from coprs.logic.complex_logic import ComplexLogic 
 13  from coprs.logic.coprs_logic import CoprChrootsLogic 
 14  from coprs.logic.packages_logic import PackagesLogic 
 15  from coprs.constants import MAX_PRIO 
 16   
 17  from coprs.views import misc 
 18  from coprs.views.backend_ns import backend_ns 
 19  from sqlalchemy.sql import false, true 
 20   
 21  import json 
 22  import logging 
 23  log = logging.getLogger(__name__) 
24 25 26 @backend_ns.route("/importing/") 27 # FIXME I'm commented 28 #@misc.backend_authenticated 29 -def dist_git_importing_queue():
30 """ 31 Return list of builds that are waiting for dist git to import the sources. 32 """ 33 builds_list = [] 34 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == false()).limit(200).all() 35 if not builds_for_import: 36 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == true()).limit(30) 37 38 for task in builds_for_import: 39 copr = task.copr 40 branches = set() 41 for b_ch in task.build_chroots: 42 branches.add(b_ch.mock_chroot.distgit_branch_name) 43 44 task_dict = { 45 "task_id": task.import_task_id, 46 "owner": copr.owner_name, 47 "project": copr.name, 48 "branches": list(branches), 49 "srpm_url": task.srpm_url, 50 } 51 if task_dict not in builds_list: 52 builds_list.append(task_dict) 53 54 response_dict = {"builds": builds_list} 55 56 return flask.jsonify(response_dict)
57 58 59 @backend_ns.route("/import-completed/", methods=["POST", "PUT"])
60 @misc.backend_authenticated 61 -def dist_git_upload_completed():
62 """ 63 Mark BuildChroot in a Build as uploaded, which means: 64 - set it to pending state 65 - set BuildChroot.git_hash 66 - if it's the last BuildChroot in a Build: 67 - delete local source 68 BuildChroot is identified with task_id which is build id + git branch name 69 - For example: 56-f22 -> build 55, chroots fedora-22-* 70 """ 71 result = {"updated": False} 72 73 if "task_id" in flask.request.json and 'branch' in flask.request.json: 74 app.logger.debug(flask.request.data) 75 task_id = flask.request.json["task_id"] 76 branch = flask.request.json["branch"] 77 build_chroots = BuildsLogic.get_buildchroots_by_build_id_and_branch(task_id, branch) 78 build = build_chroots[0].build 79 80 # Is it OK? 81 if "git_hash" in flask.request.json and "repo_name" in flask.request.json: 82 git_hash = flask.request.json["git_hash"] 83 pkg_name = flask.request.json["pkg_name"] 84 pkg_version = flask.request.json["pkg_version"] 85 86 # Now I need to assign a package to this build 87 if not PackagesLogic.get(build.copr.id, pkg_name).first(): 88 try: 89 package = PackagesLogic.add(build.copr.user, build.copr, pkg_name, build.source_type, build.source_json) 90 db.session.add(package) 91 db.session.commit() 92 except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: 93 db.session.rollback() 94 95 package = PackagesLogic.get(build.copr.id, pkg_name).first() 96 build.package_id = package.id 97 build.pkg_version = pkg_version 98 99 for ch in build_chroots: 100 if ch.status == helpers.StatusEnum("importing"): 101 ch.status = helpers.StatusEnum("pending") 102 ch.priority = (BuildsLogic.get_task_lowest_priority(build.is_background)+1)%MAX_PRIO 103 ch.git_hash = git_hash 104 105 # Failed? 106 elif "error" in flask.request.json: 107 error_type = flask.request.json["error"] 108 109 try: 110 build.fail_type = helpers.FailTypeEnum(error_type) 111 except KeyError: 112 build.fail_type = helpers.FailTypeEnum("unknown_error") 113 114 for ch in build_chroots: 115 ch.status = helpers.StatusEnum("failed") 116 117 # is it the last chroot? 118 if not build.has_importing_chroot: 119 BuildsLogic.delete_local_source(build) 120 121 db.session.commit() 122 123 result.update({"updated": True}) 124 125 return flask.jsonify(result)
126
127 128 -def get_build_record(task):
129 if not task: 130 return None 131 132 build_config = helpers.generate_build_config(task.build.copr, task.mock_chroot.name) 133 build_record = None 134 try: 135 build_record = { 136 "task_id": task.task_id, 137 "build_id": task.build.id, 138 "project_owner": task.build.copr.owner_name, 139 "project_name": task.build.copr.name, 140 "submitter": task.build.user.name if task.build.user else None, # there is no user for webhook builds 141 "chroot": task.mock_chroot.name, 142 143 "repos": task.build.repos, 144 "memory_reqs": task.build.memory_reqs, 145 "timeout": task.build.timeout, 146 "enable_net": task.build.enable_net, 147 "git_repo": task.build.package.dist_git_repo, 148 "git_hash": task.git_hash, 149 "source_type": helpers.BuildSourceEnum("scm"), 150 "source_json": json.dumps( 151 {'clone_url': task.build.package.dist_git_clone_url, 'committish': task.git_hash}), 152 153 "package_name": task.build.package.name, 154 "package_version": task.build.pkg_version, 155 "repos": build_config.get("repos"), 156 "buildroot_pkgs": build_config.get("additional_packages"), 157 "use_bootstrap_container": build_config.get("use_bootstrap_container") 158 } 159 160 except Exception as err: 161 app.logger.exception(err) 162 163 return build_record
164
165 166 -def get_srpm_build_record(task):
167 if not task: 168 return None 169 170 try: 171 build_record = { 172 "build_id": task.id, 173 "project_owner": task.copr.owner_name, 174 "project_name": task.copr.name, 175 "source_type": task.source_type, 176 "source_json": task.source_json, 177 } 178 179 except Exception as err: 180 app.logger.exception(err) 181 182 return build_record
183
184 185 @backend_ns.route("/waiting/") 186 #@misc.backend_authenticated 187 -def waiting():
188 """ 189 Return a single action and a single build. 190 """ 191 action_record = None 192 build_record = None 193 194 action = actions_logic.ActionsLogic.get_waiting().first() 195 if action: 196 action_record = action.to_dict(options={ 197 "__columns_except__": ["result", "message", "ended_on"] 198 }) 199 200 build_task = BuildsLogic.select_build_task(background=False) 201 srpm_build_task = BuildsLogic.select_srpm_build_task(background=False) 202 203 if not build_task: 204 build_record = get_srpm_build_record(srpm_build_task) 205 elif srpm_build_task and srpm_build_task.priority < build_task.priority: 206 build_record = get_srpm_build_record(srpm_build_task) 207 else: 208 build_record = get_build_record(build_task) 209 210 if not build_record: 211 build_task = BuildsLogic.select_build_task(background=True) 212 srpm_build_task = BuildsLogic.select_srpm_build_task(background=True) 213 214 if not build_task: 215 build_record = get_srpm_build_record(srpm_build_task) 216 elif srpm_build_task and srpm_build_task.priority < build_task.priority: 217 build_record = get_srpm_build_record(srpm_build_task) 218 else: 219 build_record = get_build_record(build_task) 220 221 response_dict = {"action": action_record, "build": build_record} 222 return flask.jsonify(response_dict)
223
224 225 @backend_ns.route("/get-build-task/<task_id>") 226 -def get_build_task(task_id):
227 try: 228 task = BuildsLogic.get_build_task(task_id) 229 except exceptions.MalformedArgumentException: 230 jsonout = flask.jsonify({'msg': 'Invalid task ID'}) 231 jsonout.status_code = 500 232 return jsonout 233 except sqlalchemy.orm.exc.NoResultFound: 234 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 235 jsonout.status_code = 404 236 return jsonout 237 build_record = get_build_record(task) 238 return flask.jsonify(build_record)
239
240 241 @backend_ns.route("/get-srpm-build-task/<build_id>") 242 -def get_srpm_build_task(build_id):
243 try: 244 task = BuildsLogic.get_srpm_build_task(build_id) 245 except sqlalchemy.orm.exc.NoResultFound: 246 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 247 jsonout.status_code = 404 248 return jsonout 249 build_record = get_srpm_build_record(task) 250 return flask.jsonify(build_record)
251 252 253 @backend_ns.route("/update/", methods=["POST", "PUT"])
254 @misc.backend_authenticated 255 -def update():
256 result = {} 257 258 request_data = flask.request.json 259 for typ, logic_cls in [("actions", actions_logic.ActionsLogic), 260 ("builds", BuildsLogic)]: 261 262 if typ not in request_data: 263 continue 264 265 to_update = {} 266 for obj in request_data[typ]: 267 to_update[obj["id"]] = obj 268 269 existing = {} 270 for obj in logic_cls.get_by_ids(to_update.keys()).all(): 271 existing[obj.id] = obj 272 273 non_existing_ids = list(set(to_update.keys()) - set(existing.keys())) 274 275 for i, obj in existing.items(): 276 logic_cls.update_state_from_dict(obj, to_update[i]) 277 278 db.session.commit() 279 result.update({"updated_{0}_ids".format(typ): list(existing.keys()), 280 "non_existing_{0}_ids".format(typ): non_existing_ids}) 281 282 return flask.jsonify(result)
283 284 285 @backend_ns.route("/starting_build/", methods=["POST", "PUT"])
286 @misc.backend_authenticated 287 -def starting_build():
288 """ 289 Check if the build is not cancelled and set it to running state 290 """ 291 292 result = {"can_start": False} 293 294 if "build_id" in flask.request.json and "chroot" in flask.request.json: 295 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 296 chroot = flask.request.json.get("chroot") 297 298 if build and chroot and not build.canceled: 299 log.info("mark build {} chroot {} as starting".format(build.id, chroot)) 300 BuildsLogic.update_state_from_dict(build, { 301 "chroot": chroot, 302 "status": StatusEnum("starting") 303 }) 304 db.session.commit() 305 result["can_start"] = True 306 307 return flask.jsonify(result)
308 309 310 @backend_ns.route("/defer_build/", methods=["POST", "PUT"])
311 @misc.backend_authenticated 312 -def defer_build():
313 """ 314 Defer build (lower job priority so that other jobs can be 315 considered for building meanwhile). 316 """ 317 318 result = {"was_deferred": False} 319 320 if "build_id" in flask.request.json and "chroot" in flask.request.json: 321 build_id = flask.request.json["build_id"] 322 chroot_name = flask.request.json.get("chroot") 323 324 if build_id and chroot_name: 325 log.info("Defer build {}, chroot {}".format(build_id, chroot_name)) 326 if chroot_name == "srpm-builds": 327 task = BuildsLogic.defer(build_id) 328 else: 329 task = BuildChrootsLogic.defer(build_id, chroot_name) 330 331 result["was_deferred"] = bool(task) 332 db.session.commit() 333 334 return flask.jsonify(result)
335 336 337 @backend_ns.route("/reschedule_all_running/", methods=["POST"])
338 @misc.backend_authenticated 339 -def reschedule_all_running():
340 """ 341 Add-hoc handle. Remove after implementation of persistent task handling in copr-backend 342 """ 343 to_reschedule = \ 344 BuildsLogic.get_build_tasks(StatusEnum("starting")).all() + \ 345 BuildsLogic.get_build_tasks(StatusEnum("running")).all() 346 347 if to_reschedule: 348 for build_chroot in to_reschedule: 349 build_chroot.status = StatusEnum("pending") 350 db.session.add(build_chroot) 351 352 db.session.commit() 353 354 return "OK", 200
355 356 357 @backend_ns.route("/reschedule_build_chroot/", methods=["POST", "PUT"])
358 @misc.backend_authenticated 359 -def reschedule_build_chroot():
360 response = {} 361 if "build_id" in flask.request.json and "chroot" in flask.request.json: 362 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 363 else: 364 response["result"] = "bad request" 365 response["msg"] = "Request missing `build_id` and/or `chroot`" 366 return flask.jsonify(response) 367 368 if build: 369 if build.canceled: 370 response["result"] = "noop" 371 response["msg"] = "build was cancelled, ignoring" 372 else: 373 chroot = flask.request.json["chroot"] 374 build_chroot = build.chroots_dict_by_name.get(chroot) 375 run_statuses = set([StatusEnum("starting"), StatusEnum("running")]) 376 if build_chroot and build_chroot.status in run_statuses: 377 log.info("rescheduling build {} chroot: {}".format(build.id, build_chroot.name)) 378 BuildsLogic.update_state_from_dict(build, { 379 "chroot": chroot, 380 "status": StatusEnum("pending") 381 }) 382 db.session.commit() 383 response["result"] = "done" 384 else: 385 response["result"] = "noop" 386 response["msg"] = "build is not in running states, ignoring" 387 388 else: 389 response["result"] = "noop" 390 response["msg"] = "Build {} wasn't found".format(flask.request.json["build_id"]) 391 392 return flask.jsonify(response)
393