Package run :: Module build_on_pagure_commit
[hide private]
[frames] | no frames]

Source Code for Module run.build_on_pagure_commit

  1  #!/usr/bin/env python 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11   
 12  sys.path.append( 
 13      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 14  ) 
 15   
 16  from coprs import db, app 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18  from coprs.logic.builds_logic import BuildsLogic 
 19   
 20  TITO_TYPE = '3' 
 21  MOCK_SCM_TYPE = '4' 
 22   
 23  logging.basicConfig( 
 24      filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")), 
 25      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 26      level=logging.DEBUG) 
 27  log = logging.getLogger(__name__) 
 28   
 29  PAGURE_BASE_URL = "https://pagure.io/" 
30 31 32 -class Package(object):
33 - def build(self):
34 raise NotImplemented()
35 36 @classmethod
37 - def new_from_db_row(cls, source_type, row):
38 try: 39 return { 40 TITO_TYPE: GitAndTitoPackage, 41 MOCK_SCM_TYPE: MockSCMPackage, 42 }[source_type](row) 43 except KeyError: 44 raise Exception('Unsupported package type {}'.format(source_type))
45 46 @classmethod
47 - def get_candidates_for_rebuild(cls, source_type, clone_url_subpart):
48 if db.engine.url.drivername == "sqlite": 49 placeholder = '?' 50 true = '1' 51 else: 52 placeholder = '%s' 53 true = 'true' 54 55 rows = db.engine.execute( 56 """ 57 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 58 FROM package 59 WHERE package.source_type = '{0}' AND 60 package.webhook_rebuild = {1} AND 61 package.source_json ILIKE {placeholder} 62 """.format(source_type, true, placeholder=placeholder), '%'+clone_url_subpart+'%' 63 ) 64 return [Package.new_from_db_row(source_type, row) for row in rows]
65
66 67 -class GitAndTitoPackage(Package):
68 - def __init__(self, db_row):
69 source_json = json.loads(db_row.source_json) 70 self.pkg_id = db_row.package_id 71 self.git_url = source_json['git_url'] 72 self.git_branch = source_json['git_branch'] 73 self.git_dir = source_json['git_dir'] 74 self.tito_test = source_json['tito_test'] 75 self.copr_id = db_row.copr_id 76 self.copr = CoprsLogic.get_by_id(self.copr_id).first() 77 self.source_json = db_row.source_json
78
79 - def build(self):
80 BuildsLogic.create_new_from_tito(self.copr.user, self.copr, self.git_url, self.git_dir, self.git_branch, self.tito_test) 81 db.session.commit()
82
83 - def is_dir_in_commit(self, raw_commit_text):
84 if not self.git_dir or not raw_commit_text: 85 return True 86 87 for line in raw_commit_text.split('\n'): 88 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/.*$', line) 89 if match and match.group(2).lower() == self.git_dir.lower(): 90 return True 91 92 return False
93
94 95 -class MockSCMPackage(Package):
96 - def __init__(self, db_row):
97 source_json = json.loads(db_row.source_json) 98 self.pkg_id = db_row.package_id 99 self.scm_url = source_json['scm_url'] 100 self.scm_branch = source_json['scm_branch'] 101 self.scm_type = source_json['scm_type'] 102 self.spec = source_json['spec'] 103 self.copr_id = db_row.copr_id 104 self.copr = CoprsLogic.get_by_id(self.copr_id).first() 105 self.source_json = db_row.source_json
106
107 - def build(self):
108 BuildsLogic.create_new_from_mock(self.copr.user, self.copr, self.scm_type, self.scm_url, self.scm_branch, self.spec) 109 db.session.commit()
110
111 112 -def build_on_fedmsg_loop():
113 log.debug("Setting up poller...") 114 115 endpoint = 'tcp://hub.fedoraproject.org:9940' 116 topic = 'io.pagure.prod.pagure.git.receive' 117 118 ctx = zmq.Context() 119 s = ctx.socket(zmq.SUB) 120 s.connect(endpoint) 121 122 s.setsockopt(zmq.SUBSCRIBE, topic) 123 124 poller = zmq.Poller() 125 poller.register(s, zmq.POLLIN) 126 127 while True: 128 log.debug("Polling...") 129 evts = poller.poll(10000) 130 if not evts: 131 continue 132 133 log.debug("Receiving...") 134 topic, msg = s.recv_multipart() 135 136 log.debug("Parsing...") 137 data = json.loads(msg) 138 139 namespace = data['msg']['repo']['namespace'] 140 repo_name = data['msg']['repo']['name'] 141 branch = data['msg']['branch'] 142 start_commit = data['msg']['start_commit'] 143 end_commit = data['msg']['end_commit'] 144 145 if namespace: 146 clone_url_subpart = '/' + namespace + '/' + repo_name 147 else: 148 clone_url_subpart = '/' + repo_name 149 150 log.info("MSG:") 151 log.info("\tclone_url_subpart = {}".format(clone_url_subpart)) 152 log.info("\tbranch = {}".format(branch)) 153 154 tito_candidates = Package.get_candidates_for_rebuild(TITO_TYPE, clone_url_subpart) 155 scm_candidates = Package.get_candidates_for_rebuild(MOCK_SCM_TYPE, clone_url_subpart) 156 157 raw_commit_text = None 158 # if start_commit != end_commit, then more than one commit and no means to iterate over 159 if tito_candidates and start_commit == end_commit: 160 raw_commit_url = PAGURE_BASE_URL + clone_url_subpart + '/raw/' + start_commit 161 r = requests.get(raw_commit_url) 162 if r.status_code == requests.codes.ok: 163 raw_commit_text = r.text 164 else: 165 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url)) 166 167 for pkg in tito_candidates: 168 log.info("Considering pkg id:{}, source_json:{}".format(pkg.pkg_id, pkg.source_json)) 169 if (pkg.git_url.endswith(clone_url_subpart) or pkg.git_url.endswith(clone_url_subpart+'.git')) \ 170 and (not pkg.git_branch or branch.endswith('/'+pkg.git_branch)) \ 171 and pkg.is_dir_in_commit(raw_commit_text): 172 log.info("\t -> rebuilding.") 173 pkg.build() 174 else: 175 log.info("\t -> skipping.") 176 177 for pkg in scm_candidates: 178 log.info("Considering pkg id:{}, source_json:{}".format(pkg.pkg_id, pkg.source_json)) 179 if (pkg.scm_url.endswith(clone_url_subpart) or pkg.scm_url.endswith(clone_url_subpart+'.git')) \ 180 and (not pkg.scm_branch or branch.endswith('/'+pkg.scm_branch)): 181 log.info("\t -> rebuilding.") 182 pkg.build() 183 else: 184 log.info("\t -> skipping.")
185 186 if __name__ == '__main__': 187 while True: 188 try: 189 build_on_fedmsg_loop() 190 except: 191 log.exception("Error in fedmsg loop. Restarting it.") 192