Module manage
[hide private]
[frames] | no frames]

Source Code for Module manage

  1  #!/usr/bin/python3 
  2   
  3  import argparse 
  4  import os 
  5  import subprocess 
  6  import datetime 
  7  import sqlalchemy 
  8  import time 
  9   
 10  import flask 
 11  from flask_script import Manager, Command, Option, Group 
 12  from flask_whooshee import Whooshee 
 13   
 14  from coprs import app 
 15  from coprs import db 
 16  from coprs import exceptions 
 17  from coprs import models 
 18  from coprs.logic import coprs_logic, packages_logic, actions_logic, builds_logic 
 19  from coprs.views.misc import create_user_wrapper 
 20  from coprs.whoosheers import CoprWhoosheer 
 21  from sqlalchemy import or_ 
 22  from coprs.helpers import chroot_to_branch,StatusEnum 
 23   
 24   
25 -class TestCommand(Command):
26
27 - def run(self, test_args):
28 os.environ["COPRS_ENVIRON_UNITTEST"] = "1" 29 if not (("COPR_CONFIG" in os.environ) and os.environ["COPR_CONFIG"]): 30 os.environ["COPR_CONFIG"] = "/etc/copr/copr_unit_test.conf" 31 os.environ["PYTHONPATH"] = "." 32 return subprocess.call(["/usr/bin/python3", "-m", "pytest"] + (test_args or []))
33 34 option_list = ( 35 Option("-a", 36 dest="test_args", 37 nargs=argparse.REMAINDER), 38 )
39 40
41 -class CreateSqliteFileCommand(Command):
42 43 """ 44 Create the sqlite DB file (not the tables). 45 Used for alembic, "create_db" does this automatically. 46 """ 47
48 - def run(self):
49 if flask.current_app.config["SQLALCHEMY_DATABASE_URI"].startswith("sqlite"): 50 # strip sqlite:/// 51 datadir_name = os.path.dirname( 52 flask.current_app.config["SQLALCHEMY_DATABASE_URI"][10:]) 53 if not os.path.exists(datadir_name): 54 os.makedirs(datadir_name)
55 56
57 -class CreateDBCommand(Command):
58 59 """ 60 Create the DB schema 61 """ 62
63 - def run(self, alembic_ini=None):
64 CreateSqliteFileCommand().run() 65 db.create_all() 66 67 # load the Alembic configuration and generate the 68 # version table, "stamping" it with the most recent rev: 69 from alembic.config import Config 70 from alembic import command 71 alembic_cfg = Config(alembic_ini) 72 command.stamp(alembic_cfg, "head") 73 74 # Functions are not covered by models.py, and no migrations are run 75 # by command.stamp() above. Create functions explicitly: 76 builds_logic.BuildsLogic.init_db()
77 78 option_list = ( 79 Option("--alembic", 80 "-f", 81 dest="alembic_ini", 82 help="Path to the alembic configuration file (alembic.ini)", 83 required=True), 84 )
85 86
87 -class DropDBCommand(Command):
88 89 """ 90 Delete DB 91 """ 92
93 - def run(self):
94 db.drop_all()
95 96
97 -class ChrootCommand(Command):
98
99 - def print_invalid_format(self, chroot_name):
100 print( 101 "{0} - invalid chroot format, must be '{release}-{version}-{arch}'." 102 .format(chroot_name))
103
104 - def print_already_exists(self, chroot_name):
105 print("{0} - already exists.".format(chroot_name))
106
107 - def print_doesnt_exist(self, chroot_name):
108 print("{0} - chroot doesn\"t exist.".format(chroot_name))
109 110 option_list = ( 111 Option("chroot_names", 112 help="Chroot name, e.g. fedora-18-x86_64.", 113 nargs="+"), 114 )
115 116
117 -class CreateChrootCommand(ChrootCommand):
118 119 "Creates a mock chroot in DB" 120
121 - def __init__(self):
122 self.option_list += Option( 123 "--dist-git-branch", 124 "-b", 125 dest="branch", 126 help="Branch name for this set of new chroots"),
127
128 - def run(self, chroot_names, branch=None):
141 142
143 -class RawhideToReleaseCommand(Command):
144 145 option_list = ( 146 Option("rawhide_chroot", help="Rawhide chroot name, e.g. fedora-rawhide-x86_64."), 147 Option("dest_chroot", help="Destination chroot, e.g. fedora-24-x86_64."), 148 ) 149
150 - def run(self, rawhide_chroot, dest_chroot):
151 mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(dest_chroot).first() 152 if not mock_chroot: 153 print("Given chroot does not exist. Please run:") 154 print(" sudo python3 manage.py create_chroot {}".format(dest_chroot)) 155 return 156 157 mock_rawhide_chroot = coprs_logic.MockChrootsLogic.get_from_name(rawhide_chroot).first() 158 if not mock_rawhide_chroot: 159 print("Given rawhide chroot does not exist. Didnt you mistyped?:") 160 print(" {}".format(rawhide_chroot)) 161 return 162 163 for copr in coprs_logic.CoprsLogic.get_all(): 164 if not self.has_rawhide(copr) or not copr.follow_fedora_branching: 165 continue 166 167 self.turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) 168 169 data = {"copr": copr.name, 170 "user": copr.user.name, 171 "rawhide_chroot": rawhide_chroot, 172 "dest_chroot": dest_chroot, 173 "builds": []} 174 175 for build in builds_logic.BuildsLogic.get_multiple_by_copr(copr): 176 # rbc means rawhide_build_chroot (we needed short variable) 177 rbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, rawhide_chroot).first() 178 dbc = builds_logic.BuildChrootsLogic.get_by_build_id_and_name(build.id, dest_chroot).first() 179 180 if not rbc or rbc.status != StatusEnum("succeeded"): 181 continue 182 183 data["builds"].append(build.result_dir) 184 185 if rbc and not dbc: 186 dest_build_chroot = models.BuildChroot(**rbc.to_dict()) 187 dest_build_chroot.mock_chroot_id = mock_chroot.id 188 dest_build_chroot.mock_chroot = mock_chroot 189 dest_build_chroot.status = StatusEnum("forked") 190 db.session.add(dest_build_chroot) 191 192 if len(data["builds"]): 193 actions_logic.ActionsLogic.send_rawhide_to_release(data) 194 195 db.session.commit()
196
197 - def turn_on_the_chroot_for_copr(self, copr, rawhide_name, mock_chroot):
198 rawhide_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, rawhide_name) 199 dest_chroot = coprs_logic.CoprChrootsLogic.get_by_name_safe(copr, mock_chroot.name) 200 201 if not rawhide_chroot or dest_chroot: 202 return 203 204 create_kwargs = { 205 "buildroot_pkgs": rawhide_chroot.buildroot_pkgs, 206 "comps": rawhide_chroot.comps, 207 "comps_name": rawhide_chroot.comps_name, 208 } 209 coprs_logic.CoprChrootsLogic.create_chroot(copr.user, copr, mock_chroot, **create_kwargs)
210
211 - def has_rawhide(self, copr):
212 return any(filter(lambda ch: ch.os_version == "rawhide", copr.mock_chroots))
213 214
215 -class BackendRawhideToReleaseCommand(RawhideToReleaseCommand):
216 217 "Copy backend data of the latest successful rawhide builds into a new chroot" 218
219 - def run(self, rawhide_chroot, dest_chroot):
220 for copr in coprs_logic.CoprsLogic.get_all(): 221 if not self.has_rawhide(copr): 222 continue 223 224 data = {"copr": copr.name, 225 "user": copr.owner_name, 226 "rawhide_chroot": rawhide_chroot, 227 "dest_chroot": dest_chroot, 228 "builds": []} 229 230 for package in packages_logic.PackagesLogic.get_all(copr.id): 231 last_build = package.last_build(successful=True) 232 if last_build: 233 data["builds"].append(last_build.result_dir) 234 235 if len(data["builds"]): 236 actions_logic.ActionsLogic.send_rawhide_to_release(data) 237 print("Created copy action from {}/{} to {}/{}" 238 .format(copr.full_name, rawhide_chroot, copr.full_name, dest_chroot)) 239 240 db.session.commit()
241 242
243 -class AlterChrootCommand(ChrootCommand):
244 245 "Activates or deactivates a chroot" 246
247 - def run(self, chroot_names, action):
248 activate = (action == "activate") 249 for chroot_name in chroot_names: 250 try: 251 coprs_logic.MockChrootsLogic.edit_by_name( 252 chroot_name, activate) 253 db.session.commit() 254 except exceptions.MalformedArgumentException: 255 self.print_invalid_format(chroot_name) 256 except exceptions.NotFoundException: 257 self.print_doesnt_exist(chroot_name)
258 259 option_list = ChrootCommand.option_list + ( 260 Option("--action", 261 "-a", 262 dest="action", 263 help="Action to take - currently activate or deactivate", 264 choices=["activate", "deactivate"], 265 required=True), 266 )
267 268
269 -class DropChrootCommand(ChrootCommand):
270 271 "Activates or deactivates a chroot" 272
273 - def run(self, chroot_names):
282 283
284 -class DisplayChrootsCommand(Command):
285 286 "Displays current mock chroots" 287
288 - def run(self, active_only):
289 for ch in coprs_logic.MockChrootsLogic.get_multiple( 290 active_only=active_only).all(): 291 292 print(ch.name)
293 294 option_list = ( 295 Option("--active-only", 296 "-a", 297 dest="active_only", 298 help="Display only active chroots", 299 required=False, 300 action="store_true", 301 default=False), 302 )
303 304
305 -class AddUserCommand(Command):
306 307 """ 308 You should not use regularly as that user will not be related to FAS account. 309 This should be used only for testing or adding special accounts e.g. proxy user. 310 """ 311
312 - def run(self, name, mail, **kwargs):
313 user = models.User.query.filter(models.User.username == name).first() 314 if user: 315 print("User named {0} already exists.".format(name)) 316 return 317 318 user = create_user_wrapper(name, mail) 319 if kwargs["api_token"]: 320 user.api_token = kwargs["api_token"] 321 if kwargs["api_login"]: 322 user.api_token = kwargs["api_login"] 323 324 db.session.add(user) 325 db.session.commit()
326 327 option_list = ( 328 Option("name"), 329 Option("mail"), 330 Option("--api_token", default=None, required=False), 331 Option("--api_login", default=None, required=False), 332 )
333 334
335 -class AlterUserCommand(Command):
336
337 - def run(self, name, **kwargs):
338 user = models.User.query.filter( 339 models.User.username == name).first() 340 if not user: 341 print("No user named {0}.".format(name)) 342 return 343 344 if kwargs["admin"]: 345 user.admin = True 346 if kwargs["no_admin"]: 347 user.admin = False 348 if kwargs["proven"]: 349 user.proven = True 350 if kwargs["no_proven"]: 351 user.proven = False 352 if kwargs["proxy"]: 353 user.proxy = True 354 if kwargs["no_proxy"]: 355 user.proxy = False 356 357 db.session.add(user) 358 db.session.commit()
359 360 option_list = ( 361 Option("name"), 362 Group( 363 Option("--admin", 364 action="store_true"), 365 Option("--no-admin", 366 action="store_true"), 367 exclusive=True 368 ), 369 Group( 370 Option("--proven", 371 action="store_true"), 372 Option("--no-proven", 373 action="store_true"), 374 exclusive=True 375 ), 376 Group( 377 Option("--proxy", 378 action="store_true"), 379 Option("--no-proxy", 380 action="store_true"), 381 exclusive=True 382 ) 383 )
384 385
386 -class FailBuildCommand(Command):
387 388 """ 389 Marks build as failed on all its non-finished chroots 390 """ 391 392 option_list = [Option("build_id")] 393
394 - def run(self, build_id, **kwargs):
395 try: 396 builds_logic.BuildsLogic.mark_as_failed(build_id) 397 print("Marking non-finished chroots of build {} as failed".format(build_id)) 398 db.session.commit() 399 400 except (sqlalchemy.exc.DataError, sqlalchemy.orm.exc.NoResultFound) as e: 401 print("Error: No such build {}".format(build_id)) 402 return 1
403 404
405 -class UpdateIndexesCommand(Command):
406 """ 407 recreates whoosh indexes for all projects 408 """ 409
410 - def run(self):
411 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 412 413 writer = index.writer() 414 for copr in coprs_logic.CoprsLogic.get_all(): 415 CoprWhoosheer.delete_copr(writer, copr) 416 writer.commit(optimize=True) 417 418 writer = index.writer() 419 writer.schema = CoprWhoosheer.schema 420 writer.commit(optimize=True) 421 422 writer = index.writer() 423 for copr in coprs_logic.CoprsLogic.get_all(): 424 CoprWhoosheer.insert_copr(writer, copr) 425 writer.commit(optimize=True)
426 427
428 -class UpdateIndexesQuickCommand(Command):
429 """ 430 Recreates whoosh indexes for projects for which 431 indexed data were updated in last n minutes. 432 Doesn't update schema. 433 """ 434 435 option_list = [Option("minutes_passed")] 436
437 - def run(self, minutes_passed):
438 index = Whooshee.get_or_create_index(app, CoprWhoosheer) 439 440 writer = index.writer() 441 query = db.session.query(models.Copr).filter( 442 models.Copr.latest_indexed_data_update >= time.time()-int(minutes_passed)*60 443 ) 444 for copr in query.all(): 445 CoprWhoosheer.update_copr(writer, copr) 446 writer.commit()
447 448 449 manager = Manager(app) 450 manager.add_command("test", TestCommand()) 451 manager.add_command("create_sqlite_file", CreateSqliteFileCommand()) 452 manager.add_command("create_db", CreateDBCommand()) 453 manager.add_command("drop_db", DropDBCommand()) 454 manager.add_command("create_chroot", CreateChrootCommand()) 455 manager.add_command("alter_chroot", AlterChrootCommand()) 456 manager.add_command("display_chroots", DisplayChrootsCommand()) 457 manager.add_command("drop_chroot", DropChrootCommand()) 458 manager.add_command("alter_user", AlterUserCommand()) 459 manager.add_command("add_user", AddUserCommand()) 460 manager.add_command("fail_build", FailBuildCommand()) 461 manager.add_command("update_indexes", UpdateIndexesCommand()) 462 manager.add_command("update_indexes_quick", UpdateIndexesQuickCommand()) 463 manager.add_command("rawhide_to_release", RawhideToReleaseCommand()) 464 manager.add_command("backend_rawhide_to_release", BackendRawhideToReleaseCommand()) 465 466 if __name__ == "__main__": 467 manager.run() 468