Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4   
  5  from six import with_metaclass 
  6  from six.moves.urllib.parse import urljoin, urlparse 
  7  import pipes 
  8  from textwrap import dedent 
  9  import re 
 10   
 11  import flask 
 12  from flask import url_for 
 13  from dateutil import parser as dt_parser 
 14  from netaddr import IPAddress, IPNetwork 
 15  from redis import StrictRedis 
 16  from sqlalchemy.types import TypeDecorator, VARCHAR 
 17  import json 
 18   
 19  from coprs import constants 
 20  from coprs import app 
21 22 23 -def generate_api_token(size=30):
24 """ Generate a random string used as token to access the API 25 remotely. 26 27 :kwarg: size, the size of the token to generate, defaults to 30 28 chars. 29 :return: a string, the API token for the user. 30 """ 31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32 33 34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
38 39 40 -class CounterStatType(object):
41 REPO_DL = "repo_dl"
42
43 44 -class EnumType(type):
45
46 - def __call__(self, attr):
47 if isinstance(attr, int): 48 for k, v in self.vals.items(): 49 if v == attr: 50 return k 51 raise KeyError("num {0} is not mapped".format(attr)) 52 else: 53 return self.vals[attr]
54
55 56 -class PermissionEnum(with_metaclass(EnumType, object)):
57 vals = {"nothing": 0, "request": 1, "approved": 2} 58 59 @classmethod
60 - def choices_list(cls, without=-1):
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
63 64 -class ActionTypeEnum(with_metaclass(EnumType, object)):
65 vals = { 66 "delete": 0, 67 "rename": 1, 68 "legal-flag": 2, 69 "createrepo": 3, 70 "update_comps": 4, 71 "gen_gpg_key": 5, 72 "rawhide_to_release": 6, 73 "fork": 7, 74 "update_module_md": 8, 75 "build_module": 9, 76 "cancel_build": 10, 77 }
78
79 80 -class BackendResultEnum(with_metaclass(EnumType, object)):
81 vals = {"waiting": 0, "success": 1, "failure": 2}
82
83 84 -class RoleEnum(with_metaclass(EnumType, object)):
85 vals = {"user": 0, "admin": 1}
86
87 88 -class StatusEnum(with_metaclass(EnumType, object)):
89 vals = {"failed": 0, 90 "succeeded": 1, 91 "canceled": 2, 92 "running": 3, 93 "pending": 4, 94 "skipped": 5, # if there was this package built already 95 "starting": 6, # build picked by worker but no VM initialized 96 "importing": 7, # SRPM is being imported to dist-git 97 "forked": 8, # build(-chroot) was forked 98 }
99
100 101 -class BuildSourceEnum(with_metaclass(EnumType, object)):
102 vals = {"unset": 0, 103 "srpm_link": 1, # url 104 "srpm_upload": 2, # pkg, tmp 105 "git_and_tito": 3, # git_url, git_dir, git_branch, tito_test 106 "mock_scm": 4, # scm_type, scm_url, spec, scm_branch 107 "pypi": 5, # package_name, version, python_versions 108 "rubygems": 6, # gem_name 109 "distgit": 7, # url, branch 110 }
111
112 113 # The same enum is also in distgit's helpers.py 114 -class FailTypeEnum(with_metaclass(EnumType, object)):
115 vals = {"unset": 0, 116 # General errors mixed with errors for SRPM URL/upload: 117 "unknown_error": 1, 118 "build_error": 2, 119 "srpm_import_failed": 3, 120 "srpm_download_failed": 4, 121 "srpm_query_failed": 5, 122 "import_timeout_exceeded": 6, 123 # Git and Tito errors: 124 "tito_general_error": 30, 125 "git_clone_failed": 31, 126 "git_wrong_directory": 32, 127 "git_checkout_error": 33, 128 "srpm_build_error": 34, 129 }
130
131 132 -class JSONEncodedDict(TypeDecorator):
133 """Represents an immutable structure as a json-encoded string. 134 135 Usage:: 136 137 JSONEncodedDict(255) 138 139 """ 140 141 impl = VARCHAR 142
143 - def process_bind_param(self, value, dialect):
144 if value is not None: 145 value = json.dumps(value) 146 147 return value
148
149 - def process_result_value(self, value, dialect):
150 if value is not None: 151 value = json.loads(value) 152 return value
153
154 -class Paginator(object):
155
156 - def __init__(self, query, total_count, page=1, 157 per_page_override=None, urls_count_override=None, 158 additional_params=None):
159 160 self.query = query 161 self.total_count = total_count 162 self.page = page 163 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 164 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 165 self.additional_params = additional_params or dict() 166 167 self._sliced_query = None
168
169 - def page_slice(self, page):
170 return (self.per_page * (page - 1), 171 self.per_page * page)
172 173 @property
174 - def sliced_query(self):
175 if not self._sliced_query: 176 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 177 return self._sliced_query
178 179 @property
180 - def pages(self):
181 return int(math.ceil(self.total_count / float(self.per_page)))
182
183 - def border_url(self, request, start):
184 if start: 185 if self.page - 1 > self.urls_count / 2: 186 return self.url_for_other_page(request, 1), 1 187 else: 188 if self.page < self.pages - self.urls_count / 2: 189 return self.url_for_other_page(request, self.pages), self.pages 190 191 return None
192
193 - def get_urls(self, request):
194 left_border = self.page - self.urls_count / 2 195 left_border = 1 if left_border < 1 else left_border 196 right_border = self.page + self.urls_count / 2 197 right_border = self.pages if right_border > self.pages else right_border 198 199 return [(self.url_for_other_page(request, i), i) 200 for i in range(left_border, right_border + 1)]
201
202 - def url_for_other_page(self, request, page):
203 args = request.view_args.copy() 204 args["page"] = page 205 args.update(self.additional_params) 206 return flask.url_for(request.endpoint, **args)
207
208 209 -def chroot_to_branch(chroot):
210 """ 211 Get a git branch name from chroot. Follow the fedora naming standard. 212 """ 213 os, version, arch = chroot.split("-") 214 if os == "fedora": 215 if version == "rawhide": 216 return "master" 217 os = "f" 218 elif os == "epel" and int(version) <= 6: 219 os = "el" 220 elif os == "mageia" and version == "cauldron": 221 os = "cauldron" 222 version = "" 223 elif os == "mageia": 224 os = "mga" 225 return "{}{}".format(os, version)
226
227 228 -def branch_to_os_version(branch):
229 os = None 230 version = None 231 if branch == "master": 232 os = "fedora" 233 version = "rawhide" 234 elif branch[0] == "f": 235 os = "fedora" 236 version = branch[1:] 237 elif branch[:4] == "epel" or branch[:2] == "el": 238 os = "epel" 239 version = branch[-1:] 240 elif branch[:6] == "custom": 241 os = "custom" 242 version = branch[-1:] 243 elif branch[:3] == "mga": 244 os = "mageia" 245 version = branch[3:] 246 elif branch[:8] == "cauldron": 247 os = "mageia" 248 version = "cauldron" 249 return os, version
250
251 252 -def splitFilename(filename):
253 """ 254 Pass in a standard style rpm fullname 255 256 Return a name, version, release, epoch, arch, e.g.:: 257 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386 258 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64 259 """ 260 261 if filename[-4:] == '.rpm': 262 filename = filename[:-4] 263 264 archIndex = filename.rfind('.') 265 arch = filename[archIndex+1:] 266 267 relIndex = filename[:archIndex].rfind('-') 268 rel = filename[relIndex+1:archIndex] 269 270 verIndex = filename[:relIndex].rfind('-') 271 ver = filename[verIndex+1:relIndex] 272 273 epochIndex = filename.find(':') 274 if epochIndex == -1: 275 epoch = '' 276 else: 277 epoch = filename[:epochIndex] 278 279 name = filename[epochIndex + 1:verIndex] 280 return name, ver, rel, epoch, arch
281
282 283 -def parse_package_name(pkg):
284 """ 285 Parse package name from possibly incomplete nvra string. 286 """ 287 288 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 289 return splitFilename(pkg)[0] 290 291 # doesn"t seem like valid pkg string, try to guess package name 292 result = "" 293 pkg = pkg.replace(".rpm", "").replace(".src", "") 294 295 for delim in ["-", "."]: 296 if delim in pkg: 297 parts = pkg.split(delim) 298 for part in parts: 299 if any(map(lambda x: x.isdigit(), part)): 300 return result[:-1] 301 302 result += part + "-" 303 304 return result[:-1] 305 306 return pkg
307
308 309 -def generate_repo_url(mock_chroot, url):
310 """ Generates url with build results for .repo file. 311 No checks if copr or mock_chroot exists. 312 """ 313 if mock_chroot.os_release == "fedora": 314 if mock_chroot.os_version != "rawhide": 315 mock_chroot.os_version = "$releasever" 316 317 url = urljoin( 318 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 319 mock_chroot.os_version, "$basearch")) 320 321 return url
322
323 324 -def fix_protocol_for_backend(url):
325 """ 326 Ensure that url either has http or https protocol according to the 327 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 328 """ 329 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 330 return url.replace("http://", "https://") 331 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 332 return url.replace("https://", "http://") 333 else: 334 return url
335
336 337 -def fix_protocol_for_frontend(url):
338 """ 339 Ensure that url either has http or https protocol according to the 340 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 341 """ 342 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 343 return url.replace("http://", "https://") 344 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 345 return url.replace("https://", "http://") 346 else: 347 return url
348
349 350 -class Serializer(object):
351
352 - def to_dict(self, options=None):
353 """ 354 Usage: 355 356 SQLAlchObject.to_dict() => returns a flat dict of the object 357 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 358 and will include a flat dict of object foo inside of that 359 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 360 a dict of the object, which will include dict of foo 361 (which will include dict of bar) and dict of spam. 362 363 Options can also contain two special values: __columns_only__ 364 and __columns_except__ 365 366 If present, the first makes only specified fiels appear, 367 the second removes specified fields. Both of these fields 368 must be either strings (only works for one field) or lists 369 (for one and more fields). 370 371 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 372 "__columns_only__": "name"}) => 373 374 The SQLAlchObject will only put its "name" into the resulting dict, 375 while "foo" all of its fields except "id". 376 377 Options can also specify whether to include foo_id when displaying 378 related foo object (__included_ids__, defaults to True). 379 This doesn"t apply when __columns_only__ is specified. 380 """ 381 382 result = {} 383 if options is None: 384 options = {} 385 columns = self.serializable_attributes 386 387 if "__columns_only__" in options: 388 columns = options["__columns_only__"] 389 else: 390 columns = set(columns) 391 if "__columns_except__" in options: 392 columns_except = options["__columns_except__"] 393 if not isinstance(options["__columns_except__"], list): 394 columns_except = [options["__columns_except__"]] 395 396 columns -= set(columns_except) 397 398 if ("__included_ids__" in options and 399 options["__included_ids__"] is False): 400 401 related_objs_ids = [ 402 r + "_id" for r, _ in options.items() 403 if not r.startswith("__")] 404 405 columns -= set(related_objs_ids) 406 407 columns = list(columns) 408 409 for column in columns: 410 result[column] = getattr(self, column) 411 412 for related, values in options.items(): 413 if hasattr(self, related): 414 result[related] = getattr(self, related).to_dict(values) 415 return result
416 417 @property
418 - def serializable_attributes(self):
419 return map(lambda x: x.name, self.__table__.columns)
420
421 422 -class RedisConnectionProvider(object):
423 - def __init__(self, config):
424 self.host = config.get("REDIS_HOST", "127.0.0.1") 425 self.port = int(config.get("REDIS_PORT", "6379"))
426
427 - def get_connection(self):
428 return StrictRedis(host=self.host, port=self.port)
429
430 431 -def get_redis_connection():
432 """ 433 Creates connection to redis, now we use default instance at localhost, no config needed 434 """ 435 return StrictRedis()
436
437 438 -def dt_to_unixtime(dt):
439 """ 440 Converts datetime to unixtime 441 :param dt: DateTime instance 442 :rtype: float 443 """ 444 return float(dt.strftime('%s'))
445
446 447 -def string_dt_to_unixtime(dt_string):
448 """ 449 Converts datetime to unixtime from string 450 :param dt_string: datetime string 451 :rtype: str 452 """ 453 return dt_to_unixtime(dt_parser.parse(dt_string))
454
455 456 -def is_ip_from_builder_net(ip):
457 """ 458 Checks is ip is owned by the builders network 459 :param str ip: IPv4 address 460 :return bool: True 461 """ 462 ip_addr = IPAddress(ip) 463 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]): 464 if ip_addr in IPNetwork(subnet): 465 return True 466 467 return False
468
469 470 -def str2bool(v):
471 if v is None: 472 return False 473 return v.lower() in ("yes", "true", "t", "1")
474
475 476 -def copr_url(view, copr, **kwargs):
477 """ 478 Examine given copr and generate proper URL for the `view` 479 480 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 481 and therefore you should *not* pass them manually. 482 483 Usage: 484 copr_url("coprs_ns.foo", copr) 485 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 486 """ 487 if copr.is_a_group_project: 488 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 489 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
490
491 492 -def url_for_copr_view(view, group_view, copr, **kwargs):
493 if copr.is_a_group_project: 494 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 495 else: 496 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
497 498 499 from sqlalchemy.engine.default import DefaultDialect 500 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 501 502 # python2/3 compatible. 503 PY3 = str is not bytes 504 text = str if PY3 else unicode 505 int_type = int if PY3 else (int, long) 506 str_type = str if PY3 else (str, unicode)
507 508 509 -class StringLiteral(String):
510 """Teach SA how to literalize various things."""
511 - def literal_processor(self, dialect):
512 super_processor = super(StringLiteral, self).literal_processor(dialect) 513 514 def process(value): 515 if isinstance(value, int_type): 516 return text(value) 517 if not isinstance(value, str_type): 518 value = text(value) 519 result = super_processor(value) 520 if isinstance(result, bytes): 521 result = result.decode(dialect.encoding) 522 return result
523 return process
524
525 526 -class LiteralDialect(DefaultDialect):
527 colspecs = { 528 # prevent various encoding explosions 529 String: StringLiteral, 530 # teach SA about how to literalize a datetime 531 DateTime: StringLiteral, 532 # don't format py2 long integers to NULL 533 NullType: StringLiteral, 534 }
535
536 537 -def literal_query(statement):
538 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.""" 539 import sqlalchemy.orm 540 if isinstance(statement, sqlalchemy.orm.Query): 541 statement = statement.statement 542 return statement.compile( 543 dialect=LiteralDialect(), 544 compile_kwargs={'literal_binds': True}, 545 ).string
546
547 548 -def stream_template(template_name, **context):
549 app.update_template_context(context) 550 t = app.jinja_env.get_template(template_name) 551 rv = t.stream(context) 552 rv.enable_buffering(2) 553 return rv
554
555 556 -def generate_repo_name(repo_url):
557 """ based on url, generate repo name """ 558 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 559 repo_url = re.sub("(__*)", '_', repo_url) 560 repo_url = re.sub("(_*$)|^_*", '', repo_url) 561 return repo_url
562
563 564 -def pre_process_repo_url(chroot, repo_url):
565 """ 566 Expands variables and sanitize repo url to be used for mock config 567 """ 568 parsed_url = urlparse(repo_url) 569 if parsed_url.scheme == "copr": 570 user = parsed_url.netloc 571 prj = parsed_url.path.split("/")[1] 572 repo_url = "/".join([ 573 flask.current_app.config["BACKEND_BASE_URL"], 574 "results", user, prj, chroot 575 ]) + "/" 576 577 repo_url = repo_url.replace("$chroot", chroot) 578 repo_url = repo_url.replace("$distname", chroot.split("-")[0]) 579 580 return pipes.quote(repo_url)
581
582 583 -def generate_build_config(copr, chroot_id):
584 """ Return dict with proper build config contents """ 585 chroot = None 586 for i in copr.copr_chroots: 587 if i.mock_chroot.name == chroot_id: 588 chroot = i 589 if not chroot: 590 return "" 591 592 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 593 594 repos = [{ 595 "id": "copr_base", 596 "url": copr.repo_url + "/{}/".format(chroot_id), 597 "name": "Copr repository", 598 }] 599 for repo in copr.repos_list: 600 repo_view = { 601 "id": generate_repo_name(repo), 602 "url": pre_process_repo_url(chroot_id, repo), 603 "name": "Additional repo " + generate_repo_name(repo), 604 } 605 repos.append(repo_view) 606 for repo in chroot.repos_list: 607 repo_view = { 608 "id": generate_repo_name(repo), 609 "url": pre_process_repo_url(chroot_id, repo), 610 "name": "Additional repo " + generate_repo_name(repo), 611 } 612 repos.append(repo_view) 613 614 return { 615 'project_id': copr.repo_id, 616 'additional_packages': packages.split(), 617 'repos': repos, 618 'chroot': chroot_id, 619 }
620