Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  from __future__ import division 
  2   
  3  import math 
  4  import random 
  5  import string 
  6   
  7  from six import with_metaclass 
  8  from six.moves.urllib.parse import urljoin, urlparse 
  9  from textwrap import dedent 
 10  import re 
 11   
 12  import flask 
 13  from flask import url_for 
 14  from dateutil import parser as dt_parser 
 15  from netaddr import IPAddress, IPNetwork 
 16  from redis import StrictRedis 
 17  from sqlalchemy.types import TypeDecorator, VARCHAR 
 18  import json 
 19   
 20  from coprs import constants 
 21  from coprs import app 
22 23 24 -def generate_api_token(size=30):
25 """ Generate a random string used as token to access the API 26 remotely. 27 28 :kwarg: size, the size of the token to generate, defaults to 30 29 chars. 30 :return: a string, the API token for the user. 31 """ 32 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
33 34 35 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 36 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 38 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
39 40 41 -class CounterStatType(object):
42 REPO_DL = "repo_dl"
43
44 45 -class EnumType(type):
46
47 - def __call__(self, attr):
48 if isinstance(attr, int): 49 for k, v in self.vals.items(): 50 if v == attr: 51 return k 52 raise KeyError("num {0} is not mapped".format(attr)) 53 else: 54 return self.vals[attr]
55
56 57 -class PermissionEnum(with_metaclass(EnumType, object)):
58 vals = {"nothing": 0, "request": 1, "approved": 2} 59 60 @classmethod
61 - def choices_list(cls, without=-1):
62 return [(n, k) for k, n in cls.vals.items() if n != without]
63
64 65 -class ActionTypeEnum(with_metaclass(EnumType, object)):
66 vals = { 67 "delete": 0, 68 "rename": 1, 69 "legal-flag": 2, 70 "createrepo": 3, 71 "update_comps": 4, 72 "gen_gpg_key": 5, 73 "rawhide_to_release": 6, 74 "fork": 7, 75 "update_module_md": 8, 76 "build_module": 9, 77 "cancel_build": 10, 78 }
79
80 81 -class BackendResultEnum(with_metaclass(EnumType, object)):
82 vals = {"waiting": 0, "success": 1, "failure": 2}
83
84 85 -class RoleEnum(with_metaclass(EnumType, object)):
86 vals = {"user": 0, "admin": 1}
87
88 89 -class StatusEnum(with_metaclass(EnumType, object)):
90 vals = {"failed": 0, 91 "succeeded": 1, 92 "canceled": 2, 93 "running": 3, 94 "pending": 4, 95 "skipped": 5, # if there was this package built already 96 "starting": 6, # build picked by worker but no VM initialized 97 "importing": 7, # SRPM is being imported to dist-git 98 "forked": 8, # build(-chroot) was forked 99 "unknown": 1000, # order_to_status/status_to_order issue 100 }
101
102 103 -class ModuleStatusEnum(with_metaclass(EnumType, object)):
104 vals = {"pending": 0, "succeeded": 1, "failed": 2}
105
106 107 -class BuildSourceEnum(with_metaclass(EnumType, object)):
108 vals = {"unset": 0, 109 "link": 1, # url 110 "upload": 2, # pkg, tmp, url 111 "pypi": 5, # package_name, version, python_versions 112 "rubygems": 6, # gem_name 113 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 114 }
115
116 # The same enum is also in distgit's helpers.py 117 -class FailTypeEnum(with_metaclass(EnumType, object)):
118 vals = {"unset": 0, 119 # General errors mixed with errors for SRPM URL/upload: 120 "unknown_error": 1, 121 "build_error": 2, 122 "srpm_import_failed": 3, 123 "srpm_download_failed": 4, 124 "srpm_query_failed": 5, 125 "import_timeout_exceeded": 6, 126 "git_clone_failed": 31, 127 "git_wrong_directory": 32, 128 "git_checkout_error": 33, 129 "srpm_build_error": 34, 130 }
131
132 133 -class JSONEncodedDict(TypeDecorator):
134 """Represents an immutable structure as a json-encoded string. 135 136 Usage:: 137 138 JSONEncodedDict(255) 139 140 """ 141 142 impl = VARCHAR 143
144 - def process_bind_param(self, value, dialect):
145 if value is not None: 146 value = json.dumps(value) 147 148 return value
149
150 - def process_result_value(self, value, dialect):
151 if value is not None: 152 value = json.loads(value) 153 return value
154
155 -class Paginator(object):
156
157 - def __init__(self, query, total_count, page=1, 158 per_page_override=None, urls_count_override=None, 159 additional_params=None):
160 161 self.query = query 162 self.total_count = total_count 163 self.page = page 164 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 165 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 166 self.additional_params = additional_params or dict() 167 168 self._sliced_query = None
169
170 - def page_slice(self, page):
171 return (self.per_page * (page - 1), 172 self.per_page * page)
173 174 @property
175 - def sliced_query(self):
176 if not self._sliced_query: 177 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 178 return self._sliced_query
179 180 @property
181 - def pages(self):
182 return int(math.ceil(self.total_count / float(self.per_page)))
183
184 - def border_url(self, request, start):
185 if start: 186 if self.page - 1 > self.urls_count // 2: 187 return self.url_for_other_page(request, 1), 1 188 else: 189 if self.page < self.pages - self.urls_count // 2: 190 return self.url_for_other_page(request, self.pages), self.pages 191 192 return None
193
194 - def get_urls(self, request):
195 left_border = self.page - self.urls_count // 2 196 left_border = 1 if left_border < 1 else left_border 197 right_border = self.page + self.urls_count // 2 198 right_border = self.pages if right_border > self.pages else right_border 199 200 return [(self.url_for_other_page(request, i), i) 201 for i in range(left_border, right_border + 1)]
202
203 - def url_for_other_page(self, request, page):
204 args = request.view_args.copy() 205 args["page"] = page 206 args.update(self.additional_params) 207 return flask.url_for(request.endpoint, **args)
208
209 210 -def chroot_to_branch(chroot):
211 """ 212 Get a git branch name from chroot. Follow the fedora naming standard. 213 """ 214 os, version, arch = chroot.split("-") 215 if os == "fedora": 216 if version == "rawhide": 217 return "master" 218 os = "f" 219 elif os == "epel" and int(version) <= 6: 220 os = "el" 221 elif os == "mageia" and version == "cauldron": 222 os = "cauldron" 223 version = "" 224 elif os == "mageia": 225 os = "mga" 226 return "{}{}".format(os, version)
227
228 229 -def splitFilename(filename):
230 """ 231 Pass in a standard style rpm fullname 232 233 Return a name, version, release, epoch, arch, e.g.:: 234 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 235 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 236 """ 237 238 if filename[-4:] == '.rpm': 239 filename = filename[:-4] 240 241 archIndex = filename.rfind('.') 242 arch = filename[archIndex+1:] 243 244 relIndex = filename[:archIndex].rfind('-') 245 rel = filename[relIndex+1:archIndex] 246 247 verIndex = filename[:relIndex].rfind('-') 248 ver = filename[verIndex+1:relIndex] 249 250 epochIndex = filename.find(':') 251 if epochIndex == -1: 252 epoch = '' 253 else: 254 epoch = filename[:epochIndex] 255 256 name = filename[epochIndex + 1:verIndex] 257 return name, ver, rel, epoch, arch
258
259 260 -def parse_package_name(pkg):
261 """ 262 Parse package name from possibly incomplete nvra string. 263 """ 264 265 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 266 return splitFilename(pkg)[0] 267 268 # doesn"t seem like valid pkg string, try to guess package name 269 result = "" 270 pkg = pkg.replace(".rpm", "").replace(".src", "") 271 272 for delim in ["-", "."]: 273 if delim in pkg: 274 parts = pkg.split(delim) 275 for part in parts: 276 if any(map(lambda x: x.isdigit(), part)): 277 return result[:-1] 278 279 result += part + "-" 280 281 return result[:-1] 282 283 return pkg
284
285 286 -def generate_repo_url(mock_chroot, url):
287 """ Generates url with build results for .repo file. 288 No checks if copr or mock_chroot exists. 289 """ 290 os_version = mock_chroot.os_version 291 292 if mock_chroot.os_release == "fedora": 293 if mock_chroot.os_version != "rawhide": 294 os_version = "$releasever" 295 296 url = urljoin( 297 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 298 os_version, "$basearch")) 299 300 return url
301
302 303 -def fix_protocol_for_backend(url):
304 """ 305 Ensure that url either has http or https protocol according to the 306 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 307 """ 308 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 309 return url.replace("http://", "https://") 310 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 311 return url.replace("https://", "http://") 312 else: 313 return url
314
315 316 -def fix_protocol_for_frontend(url):
317 """ 318 Ensure that url either has http or https protocol according to the 319 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 320 """ 321 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 322 return url.replace("http://", "https://") 323 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 324 return url.replace("https://", "http://") 325 else: 326 return url
327
328 329 -class Serializer(object):
330
331 - def to_dict(self, options=None):
332 """ 333 Usage: 334 335 SQLAlchObject.to_dict() => returns a flat dict of the object 336 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 337 and will include a flat dict of object foo inside of that 338 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 339 a dict of the object, which will include dict of foo 340 (which will include dict of bar) and dict of spam. 341 342 Options can also contain two special values: __columns_only__ 343 and __columns_except__ 344 345 If present, the first makes only specified fields appear, 346 the second removes specified fields. Both of these fields 347 must be either strings (only works for one field) or lists 348 (for one and more fields). 349 350 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 351 "__columns_only__": "name"}) => 352 353 The SQLAlchObject will only put its "name" into the resulting dict, 354 while "foo" all of its fields except "id". 355 356 Options can also specify whether to include foo_id when displaying 357 related foo object (__included_ids__, defaults to True). 358 This doesn"t apply when __columns_only__ is specified. 359 """ 360 361 result = {} 362 if options is None: 363 options = {} 364 columns = self.serializable_attributes 365 366 if "__columns_only__" in options: 367 columns = options["__columns_only__"] 368 else: 369 columns = set(columns) 370 if "__columns_except__" in options: 371 columns_except = options["__columns_except__"] 372 if not isinstance(options["__columns_except__"], list): 373 columns_except = [options["__columns_except__"]] 374 375 columns -= set(columns_except) 376 377 if ("__included_ids__" in options and 378 options["__included_ids__"] is False): 379 380 related_objs_ids = [ 381 r + "_id" for r, _ in options.items() 382 if not r.startswith("__")] 383 384 columns -= set(related_objs_ids) 385 386 columns = list(columns) 387 388 for column in columns: 389 result[column] = getattr(self, column) 390 391 for related, values in options.items(): 392 if hasattr(self, related): 393 result[related] = getattr(self, related).to_dict(values) 394 return result
395 396 @property
397 - def serializable_attributes(self):
398 return map(lambda x: x.name, self.__table__.columns)
399
400 401 -class RedisConnectionProvider(object):
402 - def __init__(self, config):
403 self.host = config.get("REDIS_HOST", "127.0.0.1") 404 self.port = int(config.get("REDIS_PORT", "6379"))
405
406 - def get_connection(self):
407 return StrictRedis(host=self.host, port=self.port)
408
409 410 -def get_redis_connection():
411 """ 412 Creates connection to redis, now we use default instance at localhost, no config needed 413 """ 414 return StrictRedis()
415
416 417 -def dt_to_unixtime(dt):
418 """ 419 Converts datetime to unixtime 420 :param dt: DateTime instance 421 :rtype: float 422 """ 423 return float(dt.strftime('%s'))
424
425 426 -def string_dt_to_unixtime(dt_string):
427 """ 428 Converts datetime to unixtime from string 429 :param dt_string: datetime string 430 :rtype: str 431 """ 432 return dt_to_unixtime(dt_parser.parse(dt_string))
433
434 435 -def is_ip_from_builder_net(ip):
436 """ 437 Checks is ip is owned by the builders network 438 :param str ip: IPv4 address 439 :return bool: True 440 """ 441 ip_addr = IPAddress(ip) 442 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 443 if ip_addr in IPNetwork(subnet): 444 return True 445 446 return False
447
448 449 -def str2bool(v):
450 if v is None: 451 return False 452 return v.lower() in ("yes", "true", "t", "1")
453
454 455 -def copr_url(view, copr, **kwargs):
456 """ 457 Examine given copr and generate proper URL for the `view` 458 459 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 460 and therefore you should *not* pass them manually. 461 462 Usage: 463 copr_url("coprs_ns.foo", copr) 464 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 465 """ 466 if copr.is_a_group_project: 467 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 468 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
469
470 471 -def url_for_copr_view(view, group_view, copr, **kwargs):
472 if copr.is_a_group_project: 473 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 474 else: 475 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
476
477 478 -def url_for_copr_builds(copr):
479 if copr.is_a_group_project: 480 return url_for("coprs_ns.group_copr_builds", 481 group_name=copr.group.name, coprname=copr.name) 482 else: 483 return url_for("coprs_ns.copr_builds", 484 username=copr.user.username, coprname=copr.name)
485 486 487 from sqlalchemy.engine.default import DefaultDialect 488 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 489 490 # python2/3 compatible. 491 PY3 = str is not bytes 492 text = str if PY3 else unicode 493 int_type = int if PY3 else (int, long) 494 str_type = str if PY3 else (str, unicode)
495 496 497 -class StringLiteral(String):
498 """Teach SA how to literalize various things."""
499 - def literal_processor(self, dialect):
500 super_processor = super(StringLiteral, self).literal_processor(dialect) 501 502 def process(value): 503 if isinstance(value, int_type): 504 return text(value) 505 if not isinstance(value, str_type): 506 value = text(value) 507 result = super_processor(value) 508 if isinstance(result, bytes): 509 result = result.decode(dialect.encoding) 510 return result
511 return process
512
513 514 -class LiteralDialect(DefaultDialect):
515 colspecs = { 516 # prevent various encoding explosions 517 String: StringLiteral, 518 # teach SA about how to literalize a datetime 519 DateTime: StringLiteral, 520 # don't format py2 long integers to NULL 521 NullType: StringLiteral, 522 }
523
524 525 -def literal_query(statement):
526 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 527 import sqlalchemy.orm 528 if isinstance(statement, sqlalchemy.orm.Query): 529 statement = statement.statement 530 return statement.compile( 531 dialect=LiteralDialect(), 532 compile_kwargs={'literal_binds': True}, 533 ).string
534
535 536 -def stream_template(template_name, **context):
537 app.update_template_context(context) 538 t = app.jinja_env.get_template(template_name) 539 rv = t.stream(context) 540 rv.enable_buffering(2) 541 return rv
542
543 544 -def generate_repo_name(repo_url):
545 """ based on url, generate repo name """ 546 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 547 repo_url = re.sub("(__*)", '_', repo_url) 548 repo_url = re.sub("(_*$)|^_*", '', repo_url) 549 return repo_url
550
551 552 -def pre_process_repo_url(chroot, repo_url):
553 """ 554 Expands variables and sanitize repo url to be used for mock config 555 """ 556 parsed_url = urlparse(repo_url) 557 if parsed_url.scheme == "copr": 558 user = parsed_url.netloc 559 prj = parsed_url.path.split("/")[1] 560 repo_url = "/".join([ 561 flask.current_app.config["BACKEND_BASE_URL"], 562 "results", user, prj, chroot 563 ]) + "/" 564 565 repo_url = repo_url.replace("$chroot", chroot) 566 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 567 return repo_url
568
569 570 -def generate_build_config(copr, chroot_id):
571 """ Return dict with proper build config contents """ 572 chroot = None 573 for i in copr.copr_chroots: 574 if i.mock_chroot.name == chroot_id: 575 chroot = i 576 if not chroot: 577 return {} 578 579 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 580 581 repos = [{ 582 "id": "copr_base", 583 "url": copr.repo_url + "/{}/".format(chroot_id), 584 "name": "Copr repository", 585 }] 586 587 if not copr.auto_createrepo: 588 repos.append({ 589 "id": "copr_base_devel", 590 "url": copr.repo_url + "/{}/devel/".format(chroot_id), 591 "name": "Copr buildroot", 592 }) 593 594 for repo in copr.repos_list: 595 repo_view = { 596 "id": generate_repo_name(repo), 597 "url": pre_process_repo_url(chroot_id, repo), 598 "name": "Additional repo " + generate_repo_name(repo), 599 } 600 repos.append(repo_view) 601 for repo in chroot.repos_list: 602 repo_view = { 603 "id": generate_repo_name(repo), 604 "url": pre_process_repo_url(chroot_id, repo), 605 "name": "Additional repo " + generate_repo_name(repo), 606 } 607 repos.append(repo_view) 608 609 return { 610 'project_id': copr.repo_id, 611 'additional_packages': packages.split(), 612 'repos': repos, 613 'chroot': chroot_id, 614 'use_bootstrap_container': copr.use_bootstrap_container 615 }
616