1 import math
2 import random
3 import string
4
5 from six import with_metaclass
6 from six.moves.urllib.parse import urljoin, urlparse, parse_qs
7 from textwrap import dedent
8 import re
9
10 import flask
11 import posixpath
12 from flask import url_for
13 from dateutil import parser as dt_parser
14 from netaddr import IPAddress, IPNetwork
15 from redis import StrictRedis
16 from sqlalchemy.types import TypeDecorator, VARCHAR
17 import json
18
19 from coprs import constants
20 from coprs import app
24 """ Generate a random string used as token to access the API
25 remotely.
26
27 :kwarg: size, the size of the token to generate, defaults to 30
28 chars.
29 :return: a string, the API token for the user.
30 """
31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32
33
34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
42
45
47 if isinstance(attr, int):
48 for k, v in self.vals.items():
49 if v == attr:
50 return k
51 raise KeyError("num {0} is not mapped".format(attr))
52 else:
53 return self.vals[attr]
54
57 vals = {"nothing": 0, "request": 1, "approved": 2}
58
59 @classmethod
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
65 vals = {
66 "delete": 0,
67 "legal-flag": 2,
68 "createrepo": 3,
69 "update_comps": 4,
70 "gen_gpg_key": 5,
71 "rawhide_to_release": 6,
72 "fork": 7,
73 "update_module_md": 8,
74 "build_module": 9,
75 "cancel_build": 10,
76 }
77
80 vals = {"waiting": 0, "success": 1, "failure": 2}
81
82
83 -class RoleEnum(with_metaclass(EnumType, object)):
84 vals = {"user": 0, "admin": 1}
85
86
87 -class StatusEnum(with_metaclass(EnumType, object)):
88 vals = {
89 "failed": 0,
90 "succeeded": 1,
91 "canceled": 2,
92 "running": 3,
93 "pending": 4,
94 "skipped": 5,
95 "starting": 6,
96 "importing": 7,
97 "forked": 8,
98 "waiting": 9,
99 "unknown": 1000,
100 }
101
104 vals = {"pending": 0, "succeeded": 1, "failed": 2}
105
108 vals = {"unset": 0,
109 "link": 1,
110 "upload": 2,
111 "pypi": 5,
112 "rubygems": 6,
113 "scm": 8,
114 "custom": 9,
115 }
116
119 vals = {"unset": 0,
120
121 "unknown_error": 1,
122 "build_error": 2,
123 "srpm_import_failed": 3,
124 "srpm_download_failed": 4,
125 "srpm_query_failed": 5,
126 "import_timeout_exceeded": 6,
127 "git_clone_failed": 31,
128 "git_wrong_directory": 32,
129 "git_checkout_error": 33,
130 "srpm_build_error": 34,
131 }
132
135 """Represents an immutable structure as a json-encoded string.
136
137 Usage::
138
139 JSONEncodedDict(255)
140
141 """
142
143 impl = VARCHAR
144
146 if value is not None:
147 value = json.dumps(value)
148
149 return value
150
152 if value is not None:
153 value = json.loads(value)
154 return value
155
158 - def __init__(self, query, total_count, page=1,
159 per_page_override=None, urls_count_override=None,
160 additional_params=None):
161
162 self.query = query
163 self.total_count = total_count
164 self.page = page
165 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
166 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
167 self.additional_params = additional_params or dict()
168
169 self._sliced_query = None
170
171 - def page_slice(self, page):
172 return (self.per_page * (page - 1),
173 self.per_page * page)
174
175 @property
177 if not self._sliced_query:
178 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
179 return self._sliced_query
180
181 @property
183 return int(math.ceil(self.total_count / float(self.per_page)))
184
186 if start:
187 if self.page - 1 > self.urls_count // 2:
188 return self.url_for_other_page(request, 1), 1
189 else:
190 if self.page < self.pages - self.urls_count // 2:
191 return self.url_for_other_page(request, self.pages), self.pages
192
193 return None
194
196 left_border = self.page - self.urls_count // 2
197 left_border = 1 if left_border < 1 else left_border
198 right_border = self.page + self.urls_count // 2
199 right_border = self.pages if right_border > self.pages else right_border
200
201 return [(self.url_for_other_page(request, i), i)
202 for i in range(left_border, right_border + 1)]
203
204 - def url_for_other_page(self, request, page):
205 args = request.view_args.copy()
206 args["page"] = page
207 args.update(self.additional_params)
208 return flask.url_for(request.endpoint, **args)
209
212 """
213 Get a git branch name from chroot. Follow the fedora naming standard.
214 """
215 os, version, arch = chroot.rsplit("-", 2)
216 if os == "fedora":
217 if version == "rawhide":
218 return "master"
219 os = "f"
220 elif os == "epel" and int(version) <= 6:
221 os = "el"
222 elif os == "mageia" and version == "cauldron":
223 os = "cauldron"
224 version = ""
225 elif os == "mageia":
226 os = "mga"
227 return "{}{}".format(os, version)
228
232 """
233 Pass in a standard style rpm fullname
234
235 Return a name, version, release, epoch, arch, e.g.::
236 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
237 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
238 """
239
240 if filename[-4:] == '.rpm':
241 filename = filename[:-4]
242
243 archIndex = filename.rfind('.')
244 arch = filename[archIndex+1:]
245
246 relIndex = filename[:archIndex].rfind('-')
247 rel = filename[relIndex+1:archIndex]
248
249 verIndex = filename[:relIndex].rfind('-')
250 ver = filename[verIndex+1:relIndex]
251
252 epochIndex = filename.find(':')
253 if epochIndex == -1:
254 epoch = ''
255 else:
256 epoch = filename[:epochIndex]
257
258 name = filename[epochIndex + 1:verIndex]
259 return name, ver, rel, epoch, arch
260
263 """
264 Parse package name from possibly incomplete nvra string.
265 """
266
267 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
268 return splitFilename(pkg)[0]
269
270
271 result = ""
272 pkg = pkg.replace(".rpm", "").replace(".src", "")
273
274 for delim in ["-", "."]:
275 if delim in pkg:
276 parts = pkg.split(delim)
277 for part in parts:
278 if any(map(lambda x: x.isdigit(), part)):
279 return result[:-1]
280
281 result += part + "-"
282
283 return result[:-1]
284
285 return pkg
286
310
313 """
314 Ensure that url either has http or https protocol according to the
315 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
316 """
317 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
318 return url.replace("http://", "https://")
319 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
320 return url.replace("https://", "http://")
321 else:
322 return url
323
326 """
327 Ensure that url either has http or https protocol according to the
328 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
329 """
330 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
331 return url.replace("http://", "https://")
332 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
333 return url.replace("https://", "http://")
334 else:
335 return url
336
339
341 """
342 Usage:
343
344 SQLAlchObject.to_dict() => returns a flat dict of the object
345 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
346 and will include a flat dict of object foo inside of that
347 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
348 a dict of the object, which will include dict of foo
349 (which will include dict of bar) and dict of spam.
350
351 Options can also contain two special values: __columns_only__
352 and __columns_except__
353
354 If present, the first makes only specified fields appear,
355 the second removes specified fields. Both of these fields
356 must be either strings (only works for one field) or lists
357 (for one and more fields).
358
359 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
360 "__columns_only__": "name"}) =>
361
362 The SQLAlchObject will only put its "name" into the resulting dict,
363 while "foo" all of its fields except "id".
364
365 Options can also specify whether to include foo_id when displaying
366 related foo object (__included_ids__, defaults to True).
367 This doesn"t apply when __columns_only__ is specified.
368 """
369
370 result = {}
371 if options is None:
372 options = {}
373 columns = self.serializable_attributes
374
375 if "__columns_only__" in options:
376 columns = options["__columns_only__"]
377 else:
378 columns = set(columns)
379 if "__columns_except__" in options:
380 columns_except = options["__columns_except__"]
381 if not isinstance(options["__columns_except__"], list):
382 columns_except = [options["__columns_except__"]]
383
384 columns -= set(columns_except)
385
386 if ("__included_ids__" in options and
387 options["__included_ids__"] is False):
388
389 related_objs_ids = [
390 r + "_id" for r, _ in options.items()
391 if not r.startswith("__")]
392
393 columns -= set(related_objs_ids)
394
395 columns = list(columns)
396
397 for column in columns:
398 result[column] = getattr(self, column)
399
400 for related, values in options.items():
401 if hasattr(self, related):
402 result[related] = getattr(self, related).to_dict(values)
403 return result
404
405 @property
407 return map(lambda x: x.name, self.__table__.columns)
408
412 self.host = config.get("REDIS_HOST", "127.0.0.1")
413 self.port = int(config.get("REDIS_PORT", "6379"))
414
416 return StrictRedis(host=self.host, port=self.port)
417
420 """
421 Creates connection to redis, now we use default instance at localhost, no config needed
422 """
423 return StrictRedis()
424
427 """
428 Converts datetime to unixtime
429 :param dt: DateTime instance
430 :rtype: float
431 """
432 return float(dt.strftime('%s'))
433
436 """
437 Converts datetime to unixtime from string
438 :param dt_string: datetime string
439 :rtype: str
440 """
441 return dt_to_unixtime(dt_parser.parse(dt_string))
442
445 """
446 Checks is ip is owned by the builders network
447 :param str ip: IPv4 address
448 :return bool: True
449 """
450 ip_addr = IPAddress(ip)
451 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]):
452 if ip_addr in IPNetwork(subnet):
453 return True
454
455 return False
456
459 if v is None:
460 return False
461 return v.lower() in ("yes", "true", "t", "1")
462
465 """
466 Examine given copr and generate proper URL for the `view`
467
468 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
469 and therefore you should *not* pass them manually.
470
471 Usage:
472 copr_url("coprs_ns.foo", copr)
473 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
474 """
475 if copr.is_a_group_project:
476 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
477 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
478
485
489
490
491 from sqlalchemy.engine.default import DefaultDialect
492 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
493
494
495 PY3 = str is not bytes
496 text = str if PY3 else unicode
497 int_type = int if PY3 else (int, long)
498 str_type = str if PY3 else (str, unicode)
502 """Teach SA how to literalize various things."""
515 return process
516
527
530 """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
531 import sqlalchemy.orm
532 if isinstance(statement, sqlalchemy.orm.Query):
533 statement = statement.statement
534 return statement.compile(
535 dialect=LiteralDialect(),
536 compile_kwargs={'literal_binds': True},
537 ).string
538
541 app.update_template_context(context)
542 t = app.jinja_env.get_template(template_name)
543 rv = t.stream(context)
544 rv.enable_buffering(2)
545 return rv
546
554
557 """
558 Expands variables and sanitize repo url to be used for mock config
559 """
560 parsed_url = urlparse(repo_url)
561 if parsed_url.scheme == "copr":
562 user = parsed_url.netloc
563 prj = parsed_url.path.split("/")[1]
564 repo_url = "/".join([
565 flask.current_app.config["BACKEND_BASE_URL"],
566 "results", user, prj, chroot
567 ]) + "/"
568
569 repo_url = repo_url.replace("$chroot", chroot)
570 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0])
571 return repo_url
572
575 """
576 :param repo: str repo from Copr/CoprChroot/Build/...
577 :param supported_keys list of supported optional parameters
578 :return: dict of optional parameters parsed from the repo URL
579 """
580 supported_keys = supported_keys or ["priority"]
581 if not repo.startswith("copr://"):
582 return {}
583
584 params = {}
585 qs = parse_qs(urlparse(repo).query)
586 for k, v in qs.items():
587 if k in supported_keys:
588
589
590 value = int(v[0]) if v[0].isnumeric() else v[0]
591 params[k] = value
592 return params
593
596 """ Return dict with proper build config contents """
597 chroot = None
598 for i in copr.copr_chroots:
599 if i.mock_chroot.name == chroot_id:
600 chroot = i
601 if not chroot:
602 return {}
603
604 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
605
606 repos = [{
607 "id": "copr_base",
608 "url": copr.repo_url + "/{}/".format(chroot_id),
609 "name": "Copr repository",
610 }]
611
612 if not copr.auto_createrepo:
613 repos.append({
614 "id": "copr_base_devel",
615 "url": copr.repo_url + "/{}/devel/".format(chroot_id),
616 "name": "Copr buildroot",
617 })
618
619 def get_additional_repo_views(repos_list):
620 repos = []
621 for repo in repos_list:
622 params = parse_repo_params(repo)
623 repo_view = {
624 "id": generate_repo_name(repo),
625 "url": pre_process_repo_url(chroot_id, repo),
626 "name": "Additional repo " + generate_repo_name(repo),
627 }
628 repo_view.update(params)
629 repos.append(repo_view)
630 return repos
631
632 repos.extend(get_additional_repo_views(copr.repos_list))
633 repos.extend(get_additional_repo_views(chroot.repos_list))
634
635 return {
636 'project_id': copr.repo_id,
637 'additional_packages': packages.split(),
638 'repos': repos,
639 'chroot': chroot_id,
640 'use_bootstrap_container': copr.use_bootstrap_container,
641 'with_opts': chroot.with_opts.split(),
642 'without_opts': chroot.without_opts.split(),
643 }
644
652
655 if not url:
656 return None
657
658 return re.sub(r'(\.git)?/*$', '', url)
659
662 if not url:
663 return False
664
665 url = trim_git_url(url)
666 return urlparse(url)
667