1
2
3 import subprocess
4 import argparse
5 import sys
6 import os
7 import json
8 import time
9 import re
10 import logging
11 from flask_sqlalchemy import SQLAlchemy
12 from sqlalchemy.sql import text
13
14 sys.path.append(
15 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
16 )
17
18 from coprs import db, app, helpers, models
19 from coprs.logic.builds_logic import BuildsLogic
20 from coprs.logic.coprs_logic import CoprsLogic
21
22 logging.basicConfig(
23 filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")),
24 format='[%(asctime)s][%(levelname)6s]: %(message)s',
25 level=logging.DEBUG)
26 log = logging.getLogger(__name__)
27
28 parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.')
29
30 parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'],
31 help='only check for updates from backend BACKEND, default pypi')
32 parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400,
33 help='ignore updates older than SECONDS, default 86400')
34 parser.add_argument('-v', '--version', action='version', version='1.0',
35 help='print program version and exit')
36
37 args = parser.parse_args()
38
39
41 print msg
42 log.debug(msg)
43
45 print msg
46 log.info(msg)
47
49 print >> sys.stderr, msg
50 log.error(msg)
51
53 print >> sys.stderr, msg
54 log.exception(msg)
55
57 """
58 Run given command in a subprocess
59 """
60 loginfo('Executing: '+' '.join(cmd))
61 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
62 (stdout, stderr) = process.communicate()
63 if process.returncode != 0:
64 logerror(stderr)
65 sys.exit(1)
66 return stdout
67
75
77 cmd_binary = 'curl'
78 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}'
79 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)]
80 result_json = to_json(run_cmd(get_updates_cmd))
81 messages = result_json['raw_messages']
82 pages = result_json['pages']
83
84 for p in range(2, pages+1):
85 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)]
86 result_json = to_json(run_cmd(get_updates_cmd))
87 messages += result_json['raw_messages']
88
89 return messages
90
100
102 source_type = helpers.BuildSourceEnum(args.backend.lower())
103 if db.engine.url.drivername == "sqlite":
104 placeholder = '?'
105 true = '1'
106 else:
107 placeholder = '%s'
108 true = 'true'
109 rows = db.engine.execute(
110 """
111 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id
112 FROM package
113 LEFT OUTER JOIN build ON build.package_id = package.id
114 WHERE package.source_type = {placeholder} AND
115 package.webhook_rebuild = {true} AND
116 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id));
117 """.format(placeholder=placeholder, true=true), source_type
118 )
119 return rows
120
121
125
126 - def build(self, copr, new_update_version):
128
129
134
135 - def build(self, copr, new_updated_version):
137
138
140 try:
141 return {
142 'pypi': PyPIPackage,
143 'rubygems': RubyGemsPackage,
144 }[backend](source_json)
145 except KeyError:
146 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
147
148
150 updated_packages = get_updated_packages(get_updates_messages())
151 loginfo('Updated packages according to datagrepper: {0}'.format(updated_packages))
152
153 for row in get_copr_package_info_rows():
154 source_json = json.loads(row.source_json)
155 package = package_from_source(args.backend.lower(), source_json)
156
157 latest_build_version = row.pkg_version
158 loginfo('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id))
159 if package.name in updated_packages:
160 new_updated_version = updated_packages[package.name]
161 logdebug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version))
162
163
164 if not latest_build_version or not re.match(new_updated_version, latest_build_version):
165 try:
166 copr = CoprsLogic.get_by_id(row.copr_id)[0]
167 except Exception as e:
168 logexception(e)
169 continue
170
171 loginfo('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}'
172 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id))
173 build = package.build(copr, new_updated_version)
174 db.session.commit()
175 loginfo('Launched build id {0}'.format(build.id))
176
177 if __name__ == '__main__':
178 try:
179 main()
180 except Exception as e:
181 logexception(str(e))
182