Package run :: Module check_for_anitya_version_updates
[hide private]
[frames] | no frames]

Source Code for Module run.check_for_anitya_version_updates

  1  #!/usr/bin/env python2 
  2   
  3  import subprocess 
  4  import argparse 
  5  import sys 
  6  import os 
  7  import json 
  8  import time 
  9  import re 
 10  import logging 
 11  from flask_sqlalchemy import SQLAlchemy 
 12  from sqlalchemy.sql import text 
 13   
 14  sys.path.append( 
 15      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 16  ) 
 17   
 18  from coprs import db, app, helpers, models 
 19  from coprs.logic.builds_logic import BuildsLogic 
 20  from coprs.logic.coprs_logic import CoprsLogic 
 21   
 22  logging.basicConfig( 
 23      filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")), 
 24      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 25      level=logging.DEBUG) 
 26  log = logging.getLogger(__name__) 
 27   
 28  parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.') 
 29   
 30  parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'], 
 31                     help='only check for updates from backend BACKEND, default pypi') 
 32  parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400, 
 33                     help='ignore updates older than SECONDS, default 86400') 
 34  parser.add_argument('-v', '--version', action='version', version='1.0', 
 35                     help='print program version and exit') 
 36   
 37  args = parser.parse_args() 
 38   
 39   
40 -def logdebug(msg):
41 print msg 42 log.debug(msg)
43
44 -def loginfo(msg):
45 print msg 46 log.info(msg)
47
48 -def logerror(msg):
49 print >> sys.stderr, msg 50 log.error(msg)
51
52 -def logexception(msg):
53 print >> sys.stderr, msg 54 log.exception(msg)
55
56 -def run_cmd(cmd):
57 """ 58 Run given command in a subprocess 59 """ 60 loginfo('Executing: '+' '.join(cmd)) 61 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 62 (stdout, stderr) = process.communicate() 63 if process.returncode != 0: 64 logerror(stderr) 65 sys.exit(1) 66 return stdout
67
68 -def to_json(data):
69 try: 70 data_json = json.loads(data) 71 except Exception as e: 72 loginfo(data) 73 logexception(str(e)) 74 return data_json
75
76 -def get_updates_messages():
77 cmd_binary = 'curl' 78 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}' 79 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)] 80 result_json = to_json(run_cmd(get_updates_cmd)) 81 messages = result_json['raw_messages'] 82 pages = result_json['pages'] 83 84 for p in range(2, pages+1): 85 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)] 86 result_json = to_json(run_cmd(get_updates_cmd)) 87 messages += result_json['raw_messages'] 88 89 return messages
90
91 -def get_updated_packages(updates_messages):
92 updated_packages = {} 93 for message in updates_messages: 94 update = message['msg'] 95 project = update['project'] 96 if args.backend.lower() != project['backend'].lower(): 97 continue 98 updated_packages[project['name'].lower()] = project['version'] 99 return updated_packages
100
101 -def get_copr_package_info_rows():
102 source_type = helpers.BuildSourceEnum(args.backend.lower()) 103 if db.engine.url.drivername == "sqlite": 104 placeholder = '?' 105 true = '1' 106 else: 107 placeholder = '%s' 108 true = 'true' 109 rows = db.engine.execute( 110 """ 111 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id 112 FROM package 113 LEFT OUTER JOIN build ON build.package_id = package.id 114 WHERE package.source_type = {placeholder} AND 115 package.webhook_rebuild = {true} AND 116 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id)); 117 """.format(placeholder=placeholder, true=true), source_type 118 ) 119 return rows
120 121
122 -class RubyGemsPackage(object):
123 - def __init__(self, source_json):
124 self.name = source_json['gem_name'].lower()
125
126 - def build(self, copr, new_update_version):
127 return BuildsLogic.create_new_from_rubygems(copr.user, copr, self.name, chroot_names=None)
128 129
130 -class PyPIPackage(object):
131 - def __init__(self, source_json):
132 self.name = source_json['pypi_package_name'].lower() 133 self.python_versions = source_json['python_versions']
134
135 - def build(self, copr, new_updated_version):
136 return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version, self.python_versions, chroot_names=None)
137 138
139 -def package_from_source(backend, source_json):
140 try: 141 return { 142 'pypi': PyPIPackage, 143 'rubygems': RubyGemsPackage, 144 }[backend](source_json) 145 except KeyError: 146 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
147 148
149 -def main():
150 updated_packages = get_updated_packages(get_updates_messages()) 151 loginfo('Updated packages according to datagrepper: {0}'.format(updated_packages)) 152 153 for row in get_copr_package_info_rows(): 154 source_json = json.loads(row.source_json) 155 package = package_from_source(args.backend.lower(), source_json) 156 157 latest_build_version = row.pkg_version 158 loginfo('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id)) 159 if package.name in updated_packages: 160 new_updated_version = updated_packages[package.name] 161 logdebug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version)) 162 163 # if the last build's package version is "different" from new remote package version, rebuild 164 if not latest_build_version or not re.match(new_updated_version, latest_build_version): 165 try: 166 copr = CoprsLogic.get_by_id(row.copr_id)[0] 167 except Exception as e: 168 logexception(e) 169 continue 170 171 loginfo('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}' 172 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id)) 173 build = package.build(copr, new_updated_version) 174 db.session.commit() 175 loginfo('Launched build id {0}'.format(build.id))
176 177 if __name__ == '__main__': 178 try: 179 main() 180 except Exception as e: 181 logexception(str(e)) 182