Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python3 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic, users_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from sqlalchemy import and_, or_ 
 22  from coprs.helpers import chroot_to_branch,StatusEnum 
 23   
 24   
25 -class TestCommand(Command):
26
27 - def run(self, test_args):
28 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 29 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 30 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 31 os.environ["PYTHONPATH"] = "." 32 return subprocess.call(["/usr/bin/python3", "-m", "pytest"] + (test_args or []))
33 34 option_list = ( 35 Option("-a", 36 dest="test_args", 37 nargs=argparse.REMAINDER), 38 )
39 40
41 -class CreateSqliteFileCommand(Command):
42 43 """ 44 Create the sqlite DB file (not the tables). 45 Used for alembic, "create_db" does this automatically. 46 """ 47
48 - def run(self):
49 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 50 # strip sqlite:/// 51 datadir_name = os.path.dirname( 52 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 53 if not os.path.exists(datadir_name): 54 os.makedirs(datadir_name)
55 56
57 -class CreateDBCommand(Command):
58 59 """ 60 Create the DB schema 61 """ 62
63 - def run(self, alembic_ini=None):
64 CreateSqliteFileCommand().run() 65 db.create_all() 66 67 # load the Alembic configuration and generate the 68 # version table, "stamping" it with the most recent rev: 69 from alembic.config import Config 70 from alembic import command 71 alembic_cfg = Config(alembic_ini) 72 command.stamp(alembic_cfg, "head") 73 74 # Functions are not covered by models.py, and no migrations are run 75 # by command.stamp() above. Create functions explicitly: 76 builds_logic.BuildsLogic.init_db()
77 78 option_list = ( 79 Option("--alembic", 80 "-f", 81 dest="alembic_ini", 82 help="Path to the alembic configuration file (alembic.ini)", 83 required=True), 84 )
85 86
87 -class DropDBCommand(Command):
88 89 """ 90 Delete DB 91 """ 92
93 - def run(self):
94 db.drop_all()
95 96
97 -class ChrootCommand(Command):
98
99 - def print_invalid_format(self, chroot_name):
100 print( 101 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 102 .format(chroot_name))
103
104 - def print_already_exists(self, chroot_name):
105 print("{0} - already exists.".format(chroot_name))
106
107 - def print_doesnt_exist(self, chroot_name):
108 print("{0} - chroot doesn\"t exist.".format(chroot_name))
109 110 option_list = ( 111 Option("chroot_names", 112 help="Chroot name, e.g. fedora-18-x86_64.", 113 nargs="+"), 114 )
115 116
117 -class CreateChrootCommand(ChrootCommand):
118 119 "Creates a mock chroot in DB" 120
121 - def __init__(self):
122 self.option_list += Option( 123 "--dist-git-branch", 124 "-b", 125 dest="branch", 126 help="Branch name for this set of new chroots"),
127
128 - def run(self, chroot_names, branch=None):
141 142
143 -class RawhideToReleaseCommand(Command):
144 145 option_list = ( 146 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 147 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 148 ) 149
150 - def run(self, rawhide_chroot, dest_chroot):
151 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 152 if not mock_chroot: 153 print("Given chroot does not exist. Please run:") 154 print(" sudo python3 manage.py create_chroot {}".format(dest_chroot)) 155 return 156 157 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 158 if not mock_rawhide_chroot: 159 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 160 print(" {}".format(rawhide_chroot)) 161 return 162 163 for copr in coprs_logic.CoprsLogic.get_all(): 164 if not self.has_rawhide(copr) or not copr.follow_fedora_branching: 165 continue 166 167 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 168 169 data = {"projectname": copr.name, 170 "ownername": copr.owner_name, 171 "rawhide_chroot": rawhide_chroot, 172 "dest_chroot": dest_chroot, 173 "builds": []} 174 175 for build in builds_logic.BuildsLogic.get_multiple_by_copr(copr): 176 # rbc means rawhide_build_chroot (we needed short variable) 177 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, rawhide_chroot).first() 178 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, dest_chroot).first() 179 180 if not rbc or rbc.status != StatusEnum("succeeded"): 181 continue 182 183 data["builds"].append(rbc.result_dir) 184 185 if rbc and not dbc: 186 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 187 dest_build_chroot.mock_chroot_id = mock_chroot.id 188 dest_build_chroot.mock_chroot = mock_chroot 189 dest_build_chroot.status = StatusEnum("forked") 190 db.session.add(dest_build_chroot) 191 192 if len(data["builds"]): 193 actions_logic.ActionsLogic.send_rawhide_to_release(data) 194 195 db.session.commit()
196
197 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
198 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 199 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 200 201 if not rawhide_chroot or dest_chroot: 202 return 203 204 create_kwargs = { 205 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 206 "comps": rawhide_chroot.comps, 207 "comps_name": rawhide_chroot.comps_name, 208 } 209 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
210
211 - def has_rawhide(self, copr):
212 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
213 214
215 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
216 217 "Copy backend data of the latest successful rawhide builds into a new chroot" 218
219 - def run(self, rawhide_chroot, dest_chroot):
220 for copr in coprs_logic.CoprsLogic.get_all(): 221 if not self.has_rawhide(copr): 222 continue 223 224 data = {"copr": copr.name, 225 "user": copr.owner_name, 226 "rawhide_chroot": rawhide_chroot, 227 "dest_chroot": dest_chroot, 228 "builds": []} 229 230 for package in packages_logic.PackagesLogic.get_all(copr.id): 231 last_build = package.last_build(successful=True) 232 if last_build: 233 data["builds"].append(last_build.result_dir) 234 235 if len(data["builds"]): 236 actions_logic.ActionsLogic.send_rawhide_to_release(data) 237 print("Created copy action from {}/{} to {}/{}" 238 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 239 240 db.session.commit()
241 242
243 -class AlterChrootCommand(ChrootCommand):
244 245 "Activates or deactivates a chroot" 246
247 - def run(self, chroot_names, action):
248 activate = (action == "activate") 249 for chroot_name in chroot_names: 250 try: 251 coprs_logic.MockChrootsLogic.edit_by_name( 252 chroot_name, activate) 253 db.session.commit() 254 except exceptions.MalformedArgumentException: 255 self.print_invalid_format(chroot_name) 256 except exceptions.NotFoundException: 257 self.print_doesnt_exist(chroot_name)
258 259 option_list = ChrootCommand.option_list + ( 260 Option("--action", 261 "-a", 262 dest="action", 263 help="Action to take - currently activate or deactivate", 264 choices=["activate", "deactivate"], 265 required=True), 266 )
267 268
269 -class DropChrootCommand(ChrootCommand):
270 271 "Activates or deactivates a chroot" 272
273 - def run(self, chroot_names):
282 283
284 -class DisplayChrootsCommand(Command):
285 286 "Displays current mock chroots" 287
288 - def run(self, active_only):
289 for ch in coprs_logic.MockChrootsLogic.get_multiple( 290 active_only=active_only).all(): 291 292 print(ch.name)
293 294 option_list = ( 295 Option("--active-only", 296 "-a", 297 dest="active_only", 298 help="Display only active chroots", 299 required=False, 300 action="store_true", 301 default=False), 302 )
303 304
305 -class AddUserCommand(Command):
306 307 """ 308 You should not use regularly as that user will not be related to FAS account. 309 This should be used only for testing or adding special accounts e.g. proxy user. 310 """ 311
312 - def run(self, name, mail, **kwargs):
313 user = models.User.query.filter(models.User.username == name).first() 314 if user: 315 print("User named {0} already exists.".format(name)) 316 return 317 318 user = create_user_wrapper(name, mail) 319 if kwargs["api_token"]: 320 user.api_token = kwargs["api_token"] 321 if kwargs["api_login"]: 322 user.api_token = kwargs["api_login"] 323 324 db.session.add(user) 325 db.session.commit()
326 327 option_list = ( 328 Option("name"), 329 Option("mail"), 330 Option("--api_token", default=None, required=False), 331 Option("--api_login", default=None, required=False), 332 )
333 334
335 -class DumpUserCommand(Command):
336
337 - def run(self, username):
338 user = models.User.query.filter(models.User.username == username).first() 339 if not user: 340 print("There is no user named {0}.".format(username)) 341 return 1 342 dumper = users_logic.UserDataDumper(user) 343 print(dumper.dumps(pretty=True))
344 345 option_list = ( 346 Option("username"), 347 )
348 349
350 -class AlterUserCommand(Command):
351
352 - def run(self, name, **kwargs):
353 user = models.User.query.filter( 354 models.User.username == name).first() 355 if not user: 356 print("No user named {0}.".format(name)) 357 return 358 359 if kwargs["admin"]: 360 user.admin = True 361 if kwargs["no_admin"]: 362 user.admin = False 363 if kwargs["proven"]: 364 user.proven = True 365 if kwargs["no_proven"]: 366 user.proven = False 367 if kwargs["proxy"]: 368 user.proxy = True 369 if kwargs["no_proxy"]: 370 user.proxy = False 371 372 db.session.add(user) 373 db.session.commit()
374 375 option_list = ( 376 Option("name"), 377 Group( 378 Option("--admin", 379 action="store_true"), 380 Option("--no-admin", 381 action="store_true"), 382 exclusive=True 383 ), 384 Group( 385 Option("--proven", 386 action="store_true"), 387 Option("--no-proven", 388 action="store_true"), 389 exclusive=True 390 ), 391 Group( 392 Option("--proxy", 393 action="store_true"), 394 Option("--no-proxy", 395 action="store_true"), 396 exclusive=True 397 ) 398 )
399 400
401 -class FailBuildCommand(Command):
402 403 """ 404 Marks build as failed on all its non-finished chroots 405 """ 406 407 option_list = [Option("build_id")] 408
409 - def run(self, build_id, **kwargs):
410 try: 411 builds_logic.BuildsLogic.mark_as_failed(build_id) 412 print("Marking non-finished chroots of build {} as failed".format(build_id)) 413 db.session.commit() 414 415 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 416 print("Error: No such build {}".format(build_id)) 417 return 1
418 419
420 -class UpdateIndexesCommand(Command):
421 """ 422 recreates whoosh indexes for all projects 423 """ 424
425 - def run(self):
426 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 427 428 writer = index.writer() 429 for copr in coprs_logic.CoprsLogic.get_all(): 430 CoprWhoosheer.delete_copr(writer, copr) 431 writer.commit(optimize=True) 432 433 writer = index.writer() 434 writer.schema = CoprWhoosheer.schema 435 writer.commit(optimize=True) 436 437 writer = index.writer() 438 for copr in coprs_logic.CoprsLogic.get_all(): 439 CoprWhoosheer.insert_copr(writer, copr) 440 writer.commit(optimize=True)
441 442
443 -class UpdateIndexesQuickCommand(Command):
444 """ 445 Recreates whoosh indexes for projects for which 446 indexed data were updated in last n minutes. 447 Doesn't update schema. 448 """ 449 450 option_list = [Option("minutes_passed")] 451
452 - def run(self, minutes_passed):
453 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 454 455 writer = index.writer() 456 query = db.session.query(models.Copr).filter( 457 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 458 ) 459 for copr in query.all(): 460 CoprWhoosheer.update_copr(writer, copr) 461 writer.commit()
462 463
464 -class UpdateGraphsDataCommand(Command):
465 """ 466 Generates newest graph data. 467 """ 468
469 - def run(self):
470 curr_time = int(time.time()) 471 builds_logic.BuildsLogic.get_tasks_histogram('10min', curr_time - 86599, curr_time, 600) 472 builds_logic.BuildsLogic.get_tasks_histogram('24h', curr_time - 90*86400, curr_time, 86400)
473 474
475 -class RemoveGraphsDataCommand(Command):
476 """ 477 Removes old cached graph data that is no longer used. 478 """ 479
480 - def run(self):
481 curr_time = int(time.time()) 482 models.BuildsStatistics.query.filter(or_( 483 and_(models.BuildsStatistics.time < curr_time - 91 * 86400, 484 models.BuildsStatistics.stat_type == '24h'), 485 and_(models.BuildsStatistics.time < curr_time - 87000, 486 models.BuildsStatistics.stat_type == '10min') 487 )).delete() 488 db.session.commit()
489 490 491 manager = Manager(app) 492 manager.add_command("test", TestCommand()) 493 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 494 manager.add_command("create_db", CreateDBCommand()) 495 manager.add_command("drop_db", DropDBCommand()) 496 manager.add_command("create_chroot", CreateChrootCommand()) 497 manager.add_command("alter_chroot", AlterChrootCommand()) 498 manager.add_command("display_chroots", DisplayChrootsCommand()) 499 manager.add_command("drop_chroot", DropChrootCommand()) 500 manager.add_command("alter_user", AlterUserCommand()) 501 manager.add_command("add_user", AddUserCommand()) 502 manager.add_command("dump_user", DumpUserCommand()) 503 manager.add_command("fail_build", FailBuildCommand()) 504 manager.add_command("update_indexes", UpdateIndexesCommand()) 505 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 506 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 507 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 508 manager.add_command("update_graphs", UpdateGraphsDataCommand()) 509 manager.add_command("vacuum_graphs", RemoveGraphsDataCommand()) 510 511 if __name__ == "__main__": 512 manager.run() 513