Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from run import generate_repo_packages 
 22  from sqlalchemy import or_ 
 23  from coprs.helpers import chroot_to_branch,StatusEnum 
 24   
 25   
26 -class TestCommand(Command):
27
28 - def run(self, test_args):
29 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 30 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 31 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 32 os.environ["PYTHONPATH"] = "." 33 return subprocess.call(["/usr/bin/python", "-m", "pytest"] + (test_args or []))
34 35 option_list = ( 36 Option("-a", 37 dest="test_args", 38 nargs=argparse.REMAINDER), 39 )
40 41
42 -class CreateSqliteFileCommand(Command):
43 44 """ 45 Create the sqlite DB file (not the tables). 46 Used for alembic, "create_db" does this automatically. 47 """ 48
49 - def run(self):
50 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 51 # strip sqlite:/// 52 datadir_name = os.path.dirname( 53 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 54 if not os.path.exists(datadir_name): 55 os.makedirs(datadir_name)
56 57
58 -class CreateDBCommand(Command):
59 60 """ 61 Create the DB schema 62 """ 63
64 - def run(self, alembic_ini=None):
65 CreateSqliteFileCommand().run() 66 db.create_all() 67 68 # load the Alembic configuration and generate the 69 # version table, "stamping" it with the most recent rev: 70 from alembic.config import Config 71 from alembic import command 72 alembic_cfg = Config(alembic_ini) 73 command.stamp(alembic_cfg, "head") 74 75 # Functions are not covered by models.py, and no migrations are run 76 # by command.stamp() above. Create functions explicitly: 77 builds_logic.BuildsLogic.init_db()
78 79 option_list = ( 80 Option("--alembic", 81 "-f", 82 dest="alembic_ini", 83 help="Path to the alembic configuration file (alembic.ini)", 84 required=True), 85 )
86 87
88 -class DropDBCommand(Command):
89 90 """ 91 Delete DB 92 """ 93
94 - def run(self):
95 db.drop_all()
96 97
98 -class ChrootCommand(Command):
99
100 - def print_invalid_format(self, chroot_name):
101 print( 102 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 103 .format(chroot_name))
104
105 - def print_already_exists(self, chroot_name):
106 print("{0} - already exists.".format(chroot_name))
107
108 - def print_doesnt_exist(self, chroot_name):
109 print("{0} - chroot doesn\"t exist.".format(chroot_name))
110 111 option_list = ( 112 Option("chroot_names", 113 help="Chroot name, e.g. fedora-18-x86_64.", 114 nargs="+"), 115 )
116 117
118 -class CreateChrootCommand(ChrootCommand):
119 120 "Creates a mock chroot in DB" 121
122 - def __init__(self):
123 self.option_list += Option( 124 "--dist-git-branch", 125 "-b", 126 dest="branch", 127 help="Branch name for this set of new chroots"),
128
129 - def run(self, chroot_names, branch=None):
130 for chroot_name in chroot_names: 131 if not branch: 132 branch = chroot_to_branch(chroot_name) 133 branch_object = coprs_logic.BranchesLogic.get_or_create(branch) 134 try: 135 chroot = coprs_logic.MockChrootsLogic.add(chroot_name) 136 chroot.distgit_branch = branch_object 137 db.session.commit() 138 except exceptions.MalformedArgumentException: 139 self.print_invalid_format(chroot_name) 140 except exceptions.DuplicateException: 141 self.print_already_exists(chroot_name)
142 143
144 -class RawhideToReleaseCommand(Command):
145 146 option_list = ( 147 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 148 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 149 ) 150
151 - def run(self, rawhide_chroot, dest_chroot):
152 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 153 if not mock_chroot: 154 print("Given chroot does not exist. Please run:") 155 print(" sudo python manage.py create_chroot {}".format(dest_chroot)) 156 return 157 158 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 159 if not mock_rawhide_chroot: 160 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 161 print(" {}".format(rawhide_chroot)) 162 return 163 164 for copr in coprs_logic.CoprsLogic.get_all(): 165 if not self.has_rawhide(copr) or not copr.follow_fedora_branching: 166 continue 167 168 data = {"copr": copr.name, 169 "user": copr.user.name, 170 "rawhide_chroot": rawhide_chroot, 171 "dest_chroot": dest_chroot, 172 "builds": []} 173 174 for build in builds_logic.BuildsLogic.get_multiple_by_copr(copr): 175 # rbc means rawhide_build_chroot (we needed short variable) 176 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, rawhide_chroot).first() 177 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, dest_chroot).first() 178 179 if not rbc or rbc.status != StatusEnum("succeeded"): 180 continue 181 data["builds"].append(build.result_dir_name) 182 183 if rbc and not dbc: 184 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 185 dest_build_chroot.mock_chroot_id = mock_chroot.id 186 dest_build_chroot.mock_chroot = mock_chroot 187 dest_build_chroot.status = StatusEnum("forked") 188 db.session.add(dest_build_chroot) 189 190 if len(data["builds"]): 191 actions_logic.ActionsLogic.send_rawhide_to_release(data) 192 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 193 194 db.session.commit()
195
196 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
197 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 198 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 199 200 if not rawhide_chroot or dest_chroot: 201 return 202 203 create_kwargs = { 204 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 205 "comps": rawhide_chroot.comps, 206 "comps_name": rawhide_chroot.comps_name, 207 } 208 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
209
210 - def has_rawhide(self, copr):
211 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
212 213
214 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
215 216 "Copy backend data of the latest successful rawhide builds into a new chroot" 217
218 - def run(self, rawhide_chroot, dest_chroot):
219 for copr in coprs_logic.CoprsLogic.get_all(): 220 if not self.has_rawhide(copr): 221 continue 222 223 data = {"copr": copr.name, 224 "user": copr.owner_name, 225 "rawhide_chroot": rawhide_chroot, 226 "dest_chroot": dest_chroot, 227 "builds": []} 228 229 for package in packages_logic.PackagesLogic.get_all(copr.id): 230 last_build = package.last_build(successful=True) 231 if last_build: 232 data["builds"].append(last_build.result_dir_name) 233 234 if len(data["builds"]): 235 actions_logic.ActionsLogic.send_rawhide_to_release(data) 236 print("Created copy action from {}/{} to {}/{}" 237 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 238 239 db.session.commit()
240
241 -class AlterChrootCommand(ChrootCommand):
242 243 "Activates or deactivates a chroot" 244
245 - def run(self, chroot_names, action):
246 activate = (action == "activate") 247 for chroot_name in chroot_names: 248 try: 249 coprs_logic.MockChrootsLogic.edit_by_name( 250 chroot_name, activate) 251 db.session.commit() 252 except exceptions.MalformedArgumentException: 253 self.print_invalid_format(chroot_name) 254 except exceptions.NotFoundException: 255 self.print_doesnt_exist(chroot_name)
256 257 option_list = ChrootCommand.option_list + ( 258 Option("--action", 259 "-a", 260 dest="action", 261 help="Action to take - currently activate or deactivate", 262 choices=["activate", "deactivate"], 263 required=True), 264 )
265 266
267 -class DropChrootCommand(ChrootCommand):
268 269 "Activates or deactivates a chroot" 270
271 - def run(self, chroot_names):
280 281
282 -class DisplayChrootsCommand(Command):
283 284 "Displays current mock chroots" 285
286 - def run(self, active_only):
287 for ch in coprs_logic.MockChrootsLogic.get_multiple( 288 active_only=active_only).all(): 289 290 print(ch.name)
291 292 option_list = ( 293 Option("--active-only", 294 "-a", 295 dest="active_only", 296 help="Display only active chroots", 297 required=False, 298 action="store_true", 299 default=False), 300 )
301 302
303 -class AddUserCommand(Command):
304 305 """ 306 You should not use regularly as that user will not be related to FAS account. 307 This should be used only for testing or adding special accounts e.g. proxy user. 308 """ 309
310 - def run(self, name, mail, **kwargs):
311 user = models.User.query.filter(models.User.username == name).first() 312 if user: 313 print("User named {0} already exists.".format(name)) 314 return 315 316 user = create_user_wrapper(name, mail) 317 if kwargs["api_token"]: 318 user.api_token = kwargs["api_token"] 319 if kwargs["api_login"]: 320 user.api_token = kwargs["api_login"] 321 322 db.session.add(user) 323 db.session.commit()
324 325 option_list = ( 326 Option("name"), 327 Option("mail"), 328 Option("--api_token", default=None, required=False), 329 Option("--api_login", default=None, required=False), 330 )
331 332
333 -class AlterUserCommand(Command):
334
335 - def run(self, name, **kwargs):
336 user = models.User.query.filter( 337 models.User.username == name).first() 338 if not user: 339 print("No user named {0}.".format(name)) 340 return 341 342 if kwargs["admin"]: 343 user.admin = True 344 if kwargs["no_admin"]: 345 user.admin = False 346 if kwargs["proven"]: 347 user.proven = True 348 if kwargs["no_proven"]: 349 user.proven = False 350 if kwargs["proxy"]: 351 user.proxy = True 352 if kwargs["no_proxy"]: 353 user.proxy = False 354 355 db.session.add(user) 356 db.session.commit()
357 358 option_list = ( 359 Option("name"), 360 Group( 361 Option("--admin", 362 action="store_true"), 363 Option("--no-admin", 364 action="store_true"), 365 exclusive=True 366 ), 367 Group( 368 Option("--proven", 369 action="store_true"), 370 Option("--no-proven", 371 action="store_true"), 372 exclusive=True 373 ), 374 Group( 375 Option("--proxy", 376 action="store_true"), 377 Option("--no-proxy", 378 action="store_true"), 379 exclusive=True 380 ) 381 )
382 383
384 -class FailBuildCommand(Command):
385 386 """ 387 Marks build as failed on all its non-finished chroots 388 """ 389 390 option_list = [Option("build_id")] 391
392 - def run(self, build_id, **kwargs):
393 try: 394 builds_logic.BuildsLogic.mark_as_failed(build_id) 395 print("Marking non-finished chroots of build {} as failed".format(build_id)) 396 db.session.commit() 397 398 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 399 print("Error: No such build {}".format(build_id)) 400 return 1
401 402
403 -class UpdateIndexesCommand(Command):
404 """ 405 recreates whoosh indexes for all projects 406 """ 407
408 - def run(self):
409 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 410 411 writer = index.writer() 412 for copr in coprs_logic.CoprsLogic.get_all(): 413 CoprWhoosheer.delete_copr(writer, copr) 414 writer.commit(optimize=True) 415 416 writer = index.writer() 417 writer.schema = CoprWhoosheer.schema 418 writer.commit(optimize=True) 419 420 writer = index.writer() 421 for copr in coprs_logic.CoprsLogic.get_all(): 422 CoprWhoosheer.insert_copr(writer, copr) 423 writer.commit(optimize=True)
424 425
426 -class UpdateIndexesQuickCommand(Command):
427 """ 428 Recreates whoosh indexes for projects for which 429 indexed data were updated in last n minutes. 430 Doesn't update schema. 431 """ 432 433 option_list = [Option("minutes_passed")] 434
435 - def run(self, minutes_passed):
436 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 437 438 writer = index.writer() 439 query = db.session.query(models.Copr).filter( 440 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 441 ) 442 for copr in query.all(): 443 CoprWhoosheer.update_copr(writer, copr) 444 writer.commit()
445 446
447 -class GenerateRepoPackagesCommand(Command):
448 """ 449 go through all coprs and create configuration rpm packages 450 for them, if they don't already have it 451 """ 452
453 - def run(self):
455 456 457 manager = Manager(app) 458 manager.add_command("test", TestCommand()) 459 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 460 manager.add_command("create_db", CreateDBCommand()) 461 manager.add_command("drop_db", DropDBCommand()) 462 manager.add_command("create_chroot", CreateChrootCommand()) 463 manager.add_command("alter_chroot", AlterChrootCommand()) 464 manager.add_command("display_chroots", DisplayChrootsCommand()) 465 manager.add_command("drop_chroot", DropChrootCommand()) 466 manager.add_command("alter_user", AlterUserCommand()) 467 manager.add_command("add_user", AddUserCommand()) 468 manager.add_command("fail_build", FailBuildCommand()) 469 manager.add_command("update_indexes", UpdateIndexesCommand()) 470 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 471 manager.add_command("generate_repo_packages", GenerateRepoPackagesCommand()) 472 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 473 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 474 475 if __name__ == "__main__": 476 manager.run() 477