Package coprs :: Package views :: Package backend_ns :: Module backend_general
[hide private]
[frames] | no frames]

Source Code for Module coprs.views.backend_ns.backend_general

  1  import flask 
  2  import time 
  3  import sqlalchemy 
  4   
  5  from coprs import db, app 
  6  from coprs import helpers 
  7  from coprs import models 
  8  from coprs import exceptions 
  9  from coprs.helpers import StatusEnum 
 10  from coprs.logic import actions_logic 
 11  from coprs.logic.builds_logic import BuildsLogic 
 12  from coprs.logic.complex_logic import ComplexLogic 
 13  from coprs.logic.coprs_logic import CoprChrootsLogic 
 14  from coprs.logic.packages_logic import PackagesLogic 
 15   
 16  from coprs.views import misc 
 17  from coprs.views.backend_ns import backend_ns 
 18  from sqlalchemy.sql import false, true 
 19   
 20  import json 
 21  import logging 
 22  log = logging.getLogger(__name__) 
23 24 25 @backend_ns.route("/importing/") 26 # FIXME I'm commented 27 #@misc.backend_authenticated 28 -def dist_git_importing_queue():
29 """ 30 Return list of builds that are waiting for dist git to import the sources. 31 """ 32 builds_list = [] 33 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == false()).limit(200).all() 34 if not builds_for_import: 35 builds_for_import = BuildsLogic.get_build_importing_queue().filter(models.Build.is_background == true()).limit(30) 36 37 for task in builds_for_import: 38 copr = task.copr 39 branches = set() 40 for b_ch in task.build_chroots: 41 branches.add(b_ch.mock_chroot.distgit_branch_name) 42 43 task_dict = { 44 "task_id": task.import_task_id, 45 "owner": copr.owner_name, 46 "project": copr.name, 47 "branches": list(branches), 48 "srpm_url": task.srpm_url, 49 } 50 if task_dict not in builds_list: 51 builds_list.append(task_dict) 52 53 response_dict = {"builds": builds_list} 54 55 return flask.jsonify(response_dict)
56 57 58 @backend_ns.route("/import-completed/", methods=["POST", "PUT"])
59 @misc.backend_authenticated 60 -def dist_git_upload_completed():
61 """ 62 Mark BuildChroot in a Build as uploaded, which means: 63 - set it to pending state 64 - set BuildChroot.git_hash 65 - if it's the last BuildChroot in a Build: 66 - delete local source 67 BuildChroot is identified with task_id which is build id + git branch name 68 - For example: 56-f22 -> build 55, chroots fedora-22-* 69 """ 70 result = {"updated": False} 71 72 if "task_id" in flask.request.json and 'branch' in flask.request.json: 73 app.logger.debug(flask.request.data) 74 task_id = flask.request.json["task_id"] 75 branch = flask.request.json["branch"] 76 build_chroots = BuildsLogic.get_buildchroots_by_build_id_and_branch(task_id, branch) 77 build = build_chroots[0].build 78 79 # Is it OK? 80 if "git_hash" in flask.request.json and "repo_name" in flask.request.json: 81 git_hash = flask.request.json["git_hash"] 82 pkg_name = flask.request.json["pkg_name"] 83 pkg_version = flask.request.json["pkg_version"] 84 85 # Now I need to assign a package to this build 86 if not PackagesLogic.get(build.copr.id, pkg_name).first(): 87 try: 88 package = PackagesLogic.add(build.copr.user, build.copr, pkg_name, build.source_type, build.source_json) 89 db.session.add(package) 90 db.session.commit() 91 except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: 92 db.session.rollback() 93 94 package = PackagesLogic.get(build.copr.id, pkg_name).first() 95 build.package_id = package.id 96 build.pkg_version = pkg_version 97 98 for ch in build_chroots: 99 if ch.status == helpers.StatusEnum("importing"): 100 ch.status = helpers.StatusEnum("pending") 101 ch.git_hash = git_hash 102 103 # Failed? 104 elif "error" in flask.request.json: 105 error_type = flask.request.json["error"] 106 107 try: 108 build.fail_type = helpers.FailTypeEnum(error_type) 109 except KeyError: 110 build.fail_type = helpers.FailTypeEnum("unknown_error") 111 112 for ch in build_chroots: 113 ch.status = helpers.StatusEnum("failed") 114 115 # is it the last chroot? 116 if not build.has_importing_chroot: 117 BuildsLogic.delete_local_source(build) 118 119 db.session.commit() 120 121 result.update({"updated": True}) 122 123 return flask.jsonify(result)
124
125 126 -def get_build_record(task):
127 if not task: 128 return None 129 130 build_config = helpers.generate_build_config(task.build.copr, task.mock_chroot.name) 131 build_record = None 132 try: 133 build_record = { 134 "task_id": task.task_id, 135 "build_id": task.build.id, 136 "project_owner": task.build.copr.owner_name, 137 "project_name": task.build.copr.name, 138 "submitter": task.build.user.name if task.build.user else None, # there is no user for webhook builds 139 "chroot": task.mock_chroot.name, 140 141 "repos": task.build.repos, 142 "memory_reqs": task.build.memory_reqs, 143 "timeout": task.build.timeout, 144 "enable_net": task.build.enable_net, 145 "git_repo": task.build.package.dist_git_repo, 146 "git_hash": task.git_hash, 147 "source_type": helpers.BuildSourceEnum("scm"), 148 "source_json": json.dumps( 149 {'clone_url': task.build.package.dist_git_clone_url, 'committish': task.git_hash}), 150 151 "package_name": task.build.package.name, 152 "package_version": task.build.pkg_version, 153 "repos": build_config.get("repos"), 154 "buildroot_pkgs": build_config.get("additional_packages"), 155 "use_bootstrap_container": build_config.get("use_bootstrap_container") 156 } 157 158 except Exception as err: 159 app.logger.exception(err) 160 161 return build_record
162
163 164 -def get_srpm_build_record(task):
165 if not task: 166 return None 167 168 try: 169 build_record = { 170 "build_id": task.id, 171 "project_owner": task.copr.owner_name, 172 "project_name": task.copr.name, 173 "source_type": task.source_type, 174 "source_json": task.source_json, 175 } 176 177 except Exception as err: 178 app.logger.exception(err) 179 180 return build_record
181
182 183 @backend_ns.route("/waiting/") 184 #@misc.backend_authenticated 185 -def waiting():
186 """ 187 Return a single action and a single build. 188 """ 189 action_record = None 190 build_record = None 191 192 action = actions_logic.ActionsLogic.get_waiting().first() 193 if action: 194 action_record = action.to_dict(options={ 195 "__columns_except__": ["result", "message", "ended_on"] 196 }) 197 198 task = BuildsLogic.select_build_task() 199 build_record = get_build_record(task) 200 201 if not build_record: 202 task = BuildsLogic.select_srpm_build_task() 203 build_record = get_srpm_build_record(task) 204 205 response_dict = {"action": action_record, "build": build_record} 206 return flask.jsonify(response_dict)
207
208 209 @backend_ns.route("/get-build-task/<task_id>") 210 -def get_build_task(task_id):
211 try: 212 task = BuildsLogic.get_build_task(task_id) 213 except exceptions.MalformedArgumentException: 214 jsonout = flask.jsonify({'msg': 'Invalid task ID'}) 215 jsonout.status_code = 500 216 return jsonout 217 except sqlalchemy.orm.exc.NoResultFound: 218 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 219 jsonout.status_code = 404 220 return jsonout 221 build_record = get_build_record(task) 222 return flask.jsonify(build_record)
223
224 225 @backend_ns.route("/get-srpm-build-task/<build_id>") 226 -def get_srpm_build_task(build_id):
227 try: 228 task = BuildsLogic.get_srpm_build_task(build_id) 229 except sqlalchemy.orm.exc.NoResultFound: 230 jsonout = flask.jsonify({'msg': 'Specified task ID not found'}) 231 jsonout.status_code = 404 232 return jsonout 233 build_record = get_srpm_build_record(task) 234 return flask.jsonify(build_record)
235 236 237 @backend_ns.route("/update/", methods=["POST", "PUT"])
238 @misc.backend_authenticated 239 -def update():
240 result = {} 241 242 request_data = flask.request.json 243 for typ, logic_cls in [("actions", actions_logic.ActionsLogic), 244 ("builds", BuildsLogic)]: 245 246 if typ not in request_data: 247 continue 248 249 to_update = {} 250 for obj in request_data[typ]: 251 to_update[obj["id"]] = obj 252 253 existing = {} 254 for obj in logic_cls.get_by_ids(to_update.keys()).all(): 255 existing[obj.id] = obj 256 257 non_existing_ids = list(set(to_update.keys()) - set(existing.keys())) 258 259 for i, obj in existing.items(): 260 logic_cls.update_state_from_dict(obj, to_update[i]) 261 262 db.session.commit() 263 result.update({"updated_{0}_ids".format(typ): list(existing.keys()), 264 "non_existing_{0}_ids".format(typ): non_existing_ids}) 265 266 return flask.jsonify(result)
267 268 269 @backend_ns.route("/starting_build/", methods=["POST", "PUT"])
270 @misc.backend_authenticated 271 -def starting_build():
272 """ 273 Check if the build is not cancelled and set it to running state 274 """ 275 276 result = {"can_start": False} 277 278 if "build_id" in flask.request.json and "chroot" in flask.request.json: 279 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 280 chroot = flask.request.json.get("chroot") 281 282 if build and chroot and not build.canceled: 283 log.info("mark build {} chroot {} as starting".format(build.id, chroot)) 284 BuildsLogic.update_state_from_dict(build, { 285 "chroot": chroot, 286 "status": StatusEnum("starting") 287 }) 288 db.session.commit() 289 result["can_start"] = True 290 291 return flask.jsonify(result)
292 293 294 @backend_ns.route("/defer_build/", methods=["POST", "PUT"])
295 @misc.backend_authenticated 296 -def defer_build():
297 """ 298 Defer build (keep it out of waiting jobs for some time). 299 """ 300 301 result = {"was_deferred": False} 302 303 if "build_id" in flask.request.json and "chroot" in flask.request.json: 304 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 305 chroot = flask.request.json.get("chroot") 306 307 if build and chroot: 308 log.info("Defer build {}, chroot {}".format(build.id, chroot)) 309 BuildsLogic.update_state_from_dict(build, { 310 "chroot": chroot, 311 "last_deferred": int(time.time()), 312 }) 313 db.session.commit() 314 result["was_deferred"] = True 315 316 return flask.jsonify(result)
317 318 319 @backend_ns.route("/reschedule_all_running/", methods=["POST"])
320 @misc.backend_authenticated 321 -def reschedule_all_running():
322 """ 323 Add-hoc handle. Remove after implementation of persistent task handling in copr-backend 324 """ 325 to_reschedule = \ 326 BuildsLogic.get_build_tasks(StatusEnum("starting")).all() + \ 327 BuildsLogic.get_build_tasks(StatusEnum("running")).all() 328 329 if to_reschedule: 330 for build_chroot in to_reschedule: 331 build_chroot.status = StatusEnum("pending") 332 db.session.add(build_chroot) 333 334 db.session.commit() 335 336 return "OK", 200
337 338 339 @backend_ns.route("/reschedule_build_chroot/", methods=["POST", "PUT"])
340 @misc.backend_authenticated 341 -def reschedule_build_chroot():
342 response = {} 343 if "build_id" in flask.request.json and "chroot" in flask.request.json: 344 build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) 345 else: 346 response["result"] = "bad request" 347 response["msg"] = "Request missing `build_id` and/or `chroot`" 348 return flask.jsonify(response) 349 350 if build: 351 if build.canceled: 352 response["result"] = "noop" 353 response["msg"] = "build was cancelled, ignoring" 354 else: 355 chroot = flask.request.json["chroot"] 356 build_chroot = build.chroots_dict_by_name.get(chroot) 357 run_statuses = set([StatusEnum("starting"), StatusEnum("running")]) 358 if build_chroot and build_chroot.status in run_statuses: 359 log.info("rescheduling build {} chroot: {}".format(build.id, build_chroot.name)) 360 BuildsLogic.update_state_from_dict(build, { 361 "chroot": chroot, 362 "status": StatusEnum("pending") 363 }) 364 db.session.commit() 365 response["result"] = "done" 366 else: 367 response["result"] = "noop" 368 response["msg"] = "build is not in running states, ignoring" 369 370 else: 371 response["result"] = "noop" 372 response["msg"] = "Build {} wasn't found".format(flask.request.json["build_id"]) 373 374 return flask.jsonify(response)
375