Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin, urlparse, parse_qs 
  7  from textwrap import dedent 
  8  import re 
  9   
 10  import flask 
 11  import posixpath 
 12  from flask import url_for 
 13  from dateutil import parser as dt_parser 
 14  from netaddr import IPAddress, IPNetwork 
 15  from redis import StrictRedis 
 16  from sqlalchemy.types import TypeDecorator, VARCHAR 
 17  import json 
 18   
 19  from coprs import constants 
 20  from coprs import app 
21 22 23 -def generate_api_token(size=30):
24 """ Generate a random string used as token to access the API 25 remotely. 26 27 :kwarg: size, the size of the token to generate, defaults to 30 28 chars. 29 :return: a string, the API token for the user. 30 """ 31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32 33 34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
38 39 40 -class CounterStatType(object):
41 REPO_DL = "repo_dl"
42
43 44 -class EnumType(type):
45
46 - def __call__(self, attr):
47 if isinstance(attr, int): 48 for k, v in self.vals.items(): 49 if v == attr: 50 return k 51 raise KeyError("num {0} is not mapped".format(attr)) 52 else: 53 return self.vals[attr]
54
55 56 -class PermissionEnum(with_metaclass(EnumType, object)):
57 vals = {"nothing": 0, "request": 1, "approved": 2} 58 59 @classmethod
60 - def choices_list(cls, without=-1):
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
63 64 -class ActionTypeEnum(with_metaclass(EnumType, object)):
65 vals = { 66 "delete": 0, 67 "legal-flag": 2, 68 "createrepo": 3, 69 "update_comps": 4, 70 "gen_gpg_key": 5, 71 "rawhide_to_release": 6, 72 "fork": 7, 73 "update_module_md": 8, 74 "build_module": 9, 75 "cancel_build": 10, 76 }
77
78 79 -class BackendResultEnum(with_metaclass(EnumType, object)):
80 vals = {"waiting": 0, "success": 1, "failure": 2}
81
82 83 -class RoleEnum(with_metaclass(EnumType, object)):
84 vals = {"user": 0, "admin": 1}
85
86 87 -class StatusEnum(with_metaclass(EnumType, object)):
88 vals = { 89 "failed": 0, # build failed 90 "succeeded": 1, # build succeeded 91 "canceled": 2, # build was canceled 92 "running": 3, # SRPM or RPM build is running 93 "pending": 4, # build(-chroot) is waiting to be picked 94 "skipped": 5, # if there was this package built already 95 "starting": 6, # build was picked by worker but no VM initialized yet 96 "importing": 7, # SRPM is being imported into dist-git 97 "forked": 8, # build(-chroot) was forked 98 "waiting": 9, # build(-chroot) is waiting for something else to finish 99 "unknown": 1000, # undefined 100 }
101
102 103 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
104 vals = {"pending": 0, "succeeded": 1, "failed": 2}
105
106 107 -class BuildSourceEnum(with_metaclass(EnumType, object)):
108 vals = {"unset": 0, 109 "link": 1, # url 110 "upload": 2, # pkg, tmp, url 111 "pypi": 5, # package_name, version, python_versions 112 "rubygems": 6, # gem_name 113 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 114 "custom": 9, # user-provided script to build sources 115 }
116
117 # The same enum is also in distgit's helpers.py 118 -class FailTypeEnum(with_metaclass(EnumType, object)):
119 vals = {"unset": 0, 120 # General errors mixed with errors for SRPM URL/upload: 121 "unknown_error": 1, 122 "build_error": 2, 123 "srpm_import_failed": 3, 124 "srpm_download_failed": 4, 125 "srpm_query_failed": 5, 126 "import_timeout_exceeded": 6, 127 "git_clone_failed": 31, 128 "git_wrong_directory": 32, 129 "git_checkout_error": 33, 130 "srpm_build_error": 34, 131 }
132
133 134 -class JSONEncodedDict(TypeDecorator):
135 """Represents an immutable structure as a json-encoded string. 136 137 Usage:: 138 139 JSONEncodedDict(255) 140 141 """ 142 143 impl = VARCHAR 144
145 - def process_bind_param(self, value, dialect):
146 if value is not None: 147 value = json.dumps(value) 148 149 return value
150
151 - def process_result_value(self, value, dialect):
152 if value is not None: 153 value = json.loads(value) 154 return value
155
156 157 -class Paginator(object):
158 - def __init__(self, query, total_count, page=1, 159 per_page_override=None, urls_count_override=None, 160 additional_params=None):
161 162 self.query = query 163 self.total_count = total_count 164 self.page = page 165 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 166 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 167 self.additional_params = additional_params or dict() 168 169 self._sliced_query = None
170
171 - def page_slice(self, page):
172 return (self.per_page * (page - 1), 173 self.per_page * page)
174 175 @property
176 - def sliced_query(self):
177 if not self._sliced_query: 178 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 179 return self._sliced_query
180 181 @property
182 - def pages(self):
183 return int(math.ceil(self.total_count / float(self.per_page)))
184
185 - def border_url(self, request, start):
186 if start: 187 if self.page - 1 > self.urls_count // 2: 188 return self.url_for_other_page(request, 1), 1 189 else: 190 if self.page < self.pages - self.urls_count // 2: 191 return self.url_for_other_page(request, self.pages), self.pages 192 193 return None
194
195 - def get_urls(self, request):
196 left_border = self.page - self.urls_count // 2 197 left_border = 1 if left_border < 1 else left_border 198 right_border = self.page + self.urls_count // 2 199 right_border = self.pages if right_border > self.pages else right_border 200 201 return [(self.url_for_other_page(request, i), i) 202 for i in range(left_border, right_border + 1)]
203
204 - def url_for_other_page(self, request, page):
205 args = request.view_args.copy() 206 args["page"] = page 207 args.update(self.additional_params) 208 return flask.url_for(request.endpoint, **args)
209
210 211 -def chroot_to_branch(chroot):
212 """ 213 Get a git branch name from chroot. Follow the fedora naming standard. 214 """ 215 os, version, arch = chroot.rsplit("-", 2) 216 if os == "fedora": 217 if version == "rawhide": 218 return "master" 219 os = "f" 220 elif os == "epel" and int(version) <= 6: 221 os = "el" 222 elif os == "mageia" and version == "cauldron": 223 os = "cauldron" 224 version = "" 225 elif os == "mageia": 226 os = "mga" 227 return "{}{}".format(os, version)
228
229 230 # TODO: is there something like python-rpm-utils or python-dnf-utils for this? 231 -def splitFilename(filename):
232 """ 233 Pass in a standard style rpm fullname 234 235 Return a name, version, release, epoch, arch, e.g.:: 236 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 237 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 238 """ 239 240 if filename[-4:] == '.rpm': 241 filename = filename[:-4] 242 243 archIndex = filename.rfind('.') 244 arch = filename[archIndex+1:] 245 246 relIndex = filename[:archIndex].rfind('-') 247 rel = filename[relIndex+1:archIndex] 248 249 verIndex = filename[:relIndex].rfind('-') 250 ver = filename[verIndex+1:relIndex] 251 252 epochIndex = filename.find(':') 253 if epochIndex == -1: 254 epoch = '' 255 else: 256 epoch = filename[:epochIndex] 257 258 name = filename[epochIndex + 1:verIndex] 259 return name, ver, rel, epoch, arch
260
261 262 -def parse_package_name(pkg):
263 """ 264 Parse package name from possibly incomplete nvra string. 265 """ 266 267 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 268 return splitFilename(pkg)[0] 269 270 # doesn"t seem like valid pkg string, try to guess package name 271 result = "" 272 pkg = pkg.replace(".rpm", "").replace(".src", "") 273 274 for delim in ["-", "."]: 275 if delim in pkg: 276 parts = pkg.split(delim) 277 for part in parts: 278 if any(map(lambda x: x.isdigit(), part)): 279 return result[:-1] 280 281 result += part + "-" 282 283 return result[:-1] 284 285 return pkg
286
287 288 -def generate_repo_url(mock_chroot, url):
289 """ Generates url with build results for .repo file. 290 No checks if copr or mock_chroot exists. 291 """ 292 os_version = mock_chroot.os_version 293 294 if mock_chroot.os_release == "fedora": 295 if mock_chroot.os_version != "rawhide": 296 os_version = "$releasever" 297 298 if mock_chroot.os_release == "opensuse-leap": 299 os_version = "$releasever" 300 301 if mock_chroot.os_release == "mageia": 302 if mock_chroot.os_version != "cauldron": 303 os_version = "$releasever" 304 305 url = posixpath.join( 306 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 307 os_version, "$basearch")) 308 309 return url
310
311 312 -def fix_protocol_for_backend(url):
313 """ 314 Ensure that url either has http or https protocol according to the 315 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 316 """ 317 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 318 return url.replace("http://", "https://") 319 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 320 return url.replace("https://", "http://") 321 else: 322 return url
323
324 325 -def fix_protocol_for_frontend(url):
326 """ 327 Ensure that url either has http or https protocol according to the 328 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 329 """ 330 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 331 return url.replace("http://", "https://") 332 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 333 return url.replace("https://", "http://") 334 else: 335 return url
336
337 338 -class Serializer(object):
339
340 - def to_dict(self, options=None):
341 """ 342 Usage: 343 344 SQLAlchObject.to_dict() => returns a flat dict of the object 345 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 346 and will include a flat dict of object foo inside of that 347 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 348 a dict of the object, which will include dict of foo 349 (which will include dict of bar) and dict of spam. 350 351 Options can also contain two special values: __columns_only__ 352 and __columns_except__ 353 354 If present, the first makes only specified fields appear, 355 the second removes specified fields. Both of these fields 356 must be either strings (only works for one field) or lists 357 (for one and more fields). 358 359 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 360 "__columns_only__": "name"}) => 361 362 The SQLAlchObject will only put its "name" into the resulting dict, 363 while "foo" all of its fields except "id". 364 365 Options can also specify whether to include foo_id when displaying 366 related foo object (__included_ids__, defaults to True). 367 This doesn"t apply when __columns_only__ is specified. 368 """ 369 370 result = {} 371 if options is None: 372 options = {} 373 columns = self.serializable_attributes 374 375 if "__columns_only__" in options: 376 columns = options["__columns_only__"] 377 else: 378 columns = set(columns) 379 if "__columns_except__" in options: 380 columns_except = options["__columns_except__"] 381 if not isinstance(options["__columns_except__"], list): 382 columns_except = [options["__columns_except__"]] 383 384 columns -= set(columns_except) 385 386 if ("__included_ids__" in options and 387 options["__included_ids__"] is False): 388 389 related_objs_ids = [ 390 r + "_id" for r, _ in options.items() 391 if not r.startswith("__")] 392 393 columns -= set(related_objs_ids) 394 395 columns = list(columns) 396 397 for column in columns: 398 result[column] = getattr(self, column) 399 400 for related, values in options.items(): 401 if hasattr(self, related): 402 result[related] = getattr(self, related).to_dict(values) 403 return result
404 405 @property
406 - def serializable_attributes(self):
407 return map(lambda x: x.name, self.__table__.columns)
408
409 410 -class RedisConnectionProvider(object):
411 - def __init__(self, config):
412 self.host = config.get("REDIS_HOST", "127.0.0.1") 413 self.port = int(config.get("REDIS_PORT", "6379"))
414
415 - def get_connection(self):
416 return StrictRedis(host=self.host, port=self.port)
417
418 419 -def get_redis_connection():
420 """ 421 Creates connection to redis, now we use default instance at localhost, no config needed 422 """ 423 return StrictRedis()
424
425 426 -def dt_to_unixtime(dt):
427 """ 428 Converts datetime to unixtime 429 :param dt: DateTime instance 430 :rtype: float 431 """ 432 return float(dt.strftime('%s'))
433
434 435 -def string_dt_to_unixtime(dt_string):
436 """ 437 Converts datetime to unixtime from string 438 :param dt_string: datetime string 439 :rtype: str 440 """ 441 return dt_to_unixtime(dt_parser.parse(dt_string))
442
443 444 -def is_ip_from_builder_net(ip):
445 """ 446 Checks is ip is owned by the builders network 447 :param str ip: IPv4 address 448 :return bool: True 449 """ 450 ip_addr = IPAddress(ip) 451 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 452 if ip_addr in IPNetwork(subnet): 453 return True 454 455 return False
456
457 458 -def str2bool(v):
459 if v is None: 460 return False 461 return v.lower() in ("yes", "true", "t", "1")
462
463 464 -def copr_url(view, copr, **kwargs):
465 """ 466 Examine given copr and generate proper URL for the `view` 467 468 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 469 and therefore you should *not* pass them manually. 470 471 Usage: 472 copr_url("coprs_ns.foo", copr) 473 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 474 """ 475 if copr.is_a_group_project: 476 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 477 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
478
479 480 -def url_for_copr_view(view, group_view, copr, **kwargs):
481 if copr.is_a_group_project: 482 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 483 else: 484 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
485
486 487 -def url_for_copr_builds(copr):
488 return copr_url("coprs_ns.copr_builds", copr)
489 490 491 from sqlalchemy.engine.default import DefaultDialect 492 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 493 494 # python2/3 compatible. 495 PY3 = str is not bytes 496 text = str if PY3 else unicode 497 int_type = int if PY3 else (int, long) 498 str_type = str if PY3 else (str, unicode)
499 500 501 -class StringLiteral(String):
502 """Teach SA how to literalize various things."""
503 - def literal_processor(self, dialect):
504 super_processor = super(StringLiteral, self).literal_processor(dialect) 505 506 def process(value): 507 if isinstance(value, int_type): 508 return text(value) 509 if not isinstance(value, str_type): 510 value = text(value) 511 result = super_processor(value) 512 if isinstance(result, bytes): 513 result = result.decode(dialect.encoding) 514 return result
515 return process
516
517 518 -class LiteralDialect(DefaultDialect):
519 colspecs = { 520 # prevent various encoding explosions 521 String: StringLiteral, 522 # teach SA about how to literalize a datetime 523 DateTime: StringLiteral, 524 # don't format py2 long integers to NULL 525 NullType: StringLiteral, 526 }
527
528 529 -def literal_query(statement):
530 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 531 import sqlalchemy.orm 532 if isinstance(statement, sqlalchemy.orm.Query): 533 statement = statement.statement 534 return statement.compile( 535 dialect=LiteralDialect(), 536 compile_kwargs={'literal_binds': True}, 537 ).string
538
539 540 -def stream_template(template_name, **context):
541 app.update_template_context(context) 542 t = app.jinja_env.get_template(template_name) 543 rv = t.stream(context) 544 rv.enable_buffering(2) 545 return rv
546
547 548 -def generate_repo_name(repo_url):
549 """ based on url, generate repo name """ 550 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 551 repo_url = re.sub("(__*)", '_', repo_url) 552 repo_url = re.sub("(_*$)|^_*", '', repo_url) 553 return repo_url
554
555 556 -def pre_process_repo_url(chroot, repo_url):
557 """ 558 Expands variables and sanitize repo url to be used for mock config 559 """ 560 parsed_url = urlparse(repo_url) 561 if parsed_url.scheme == "copr": 562 user = parsed_url.netloc 563 prj = parsed_url.path.split("/")[1] 564 repo_url = "/".join([ 565 flask.current_app.config["BACKEND_BASE_URL"], 566 "results", user, prj, chroot 567 ]) + "/" 568 569 repo_url = repo_url.replace("$chroot", chroot) 570 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0]) 571 return repo_url
572
573 574 -def parse_repo_params(repo, supported_keys=None):
575 """ 576 :param repo: str repo from Copr/CoprChroot/Build/... 577 :param supported_keys list of supported optional parameters 578 :return: dict of optional parameters parsed from the repo URL 579 """ 580 supported_keys = supported_keys or ["priority"] 581 if not repo.startswith("copr://"): 582 return {} 583 584 params = {} 585 qs = parse_qs(urlparse(repo).query) 586 for k, v in qs.items(): 587 if k in supported_keys: 588 # parse_qs returns values as lists, but we allow setting the param only once, 589 # so we can take just first value from it 590 value = int(v[0]) if v[0].isnumeric() else v[0] 591 params[k] = value 592 return params
593
594 595 -def generate_build_config(copr, chroot_id):
596 """ Return dict with proper build config contents """ 597 chroot = None 598 for i in copr.copr_chroots: 599 if i.mock_chroot.name == chroot_id: 600 chroot = i 601 if not chroot: 602 return {} 603 604 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 605 606 repos = [{ 607 "id": "copr_base", 608 "url": copr.repo_url + "/{}/".format(chroot_id), 609 "name": "Copr repository", 610 }] 611 612 if not copr.auto_createrepo: 613 repos.append({ 614 "id": "copr_base_devel", 615 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 616 "name": "Copr buildroot", 617 }) 618 619 def get_additional_repo_views(repos_list): 620 repos = [] 621 for repo in repos_list: 622 params = parse_repo_params(repo) 623 repo_view = { 624 "id": generate_repo_name(repo), 625 "url": pre_process_repo_url(chroot_id, repo), 626 "name": "Additional repo " + generate_repo_name(repo), 627 } 628 repo_view.update(params) 629 repos.append(repo_view) 630 return repos
631 632 repos.extend(get_additional_repo_views(copr.repos_list)) 633 repos.extend(get_additional_repo_views(chroot.repos_list)) 634 635 return { 636 'project_id': copr.repo_id, 637 'additional_packages': packages.split(), 638 'repos': repos, 639 'chroot': chroot_id, 640 'use_bootstrap_container': copr.use_bootstrap_container, 641 'with_opts': chroot.with_opts.split(), 642 'without_opts': chroot.without_opts.split(), 643 } 644
645 646 -def generate_additional_repos(copr_chroot):
647 base_repo = "copr://{}".format(copr_chroot.copr.full_name) 648 repos = [base_repo] + copr_chroot.repos_list + copr_chroot.copr.repos_list 649 if not copr_chroot.copr.auto_createrepo: 650 repos.append("copr://{}/devel".format(copr_chroot.copr.full_name)) 651 return repos
652
653 654 -def trim_git_url(url):
655 if not url: 656 return None 657 658 return re.sub(r'(\.git)?/*$', '', url)
659
660 661 -def get_parsed_git_url(url):
662 if not url: 663 return False 664 665 url = trim_git_url(url) 666 return urlparse(url)
667