Package run :: Module build_on_pagure_commit
[hide private]
[frames] | no frames]

Source Code for Module run.build_on_pagure_commit

  1  #!/usr/bin/env python 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11   
 12  sys.path.append( 
 13      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 14  ) 
 15   
 16  from coprs import db, app 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18  from coprs.logic.packages_logic import PackagesLogic 
 19  from coprs.logic.complex_logic import ComplexLogic 
 20  from coprs import helpers 
 21   
 22  try: 
 23      from urllib.parse import urlparse 
 24  except ImportError: 
 25      from urlparse import urlparse 
 26   
 27  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 28   
 29  logging.basicConfig( 
 30      filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")), 
 31      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 32      level=logging.DEBUG) 
 33  log = logging.getLogger(__name__) 
 34   
 35  PAGURE_BASE_URL = "https://pagure.io/" 
 36  PAGURE_HOSTNAME = "pagure.io" 
37 38 39 -class ScmPackage(object):
40 - def __init__(self, db_row):
41 self.pkg_id = db_row.package_id 42 self.copr_id = db_row.copr_id 43 44 self.source_json_dict = json.loads(db_row.source_json) 45 self.clone_url = self.source_json_dict.get('clone_url') or '' 46 self.committish = self.source_json_dict.get('committish') or '' 47 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 48 49 self.copr = CoprsLogic.get_by_id(self.copr_id).first() 50 self.package = ComplexLogic.get_package_by_id_safe(self.pkg_id)
51
52 - def build(self):
53 PackagesLogic.build_package(self.copr.user, self.copr, self.package) 54 db.session.commit()
55 56 @classmethod
57 - def get_candidates_for_rebuild(cls, clone_url_subpart):
58 if db.engine.url.drivername == "sqlite": 59 placeholder = '?' 60 true = '1' 61 else: 62 placeholder = '%s' 63 true = 'true' 64 65 rows = db.engine.execute( 66 """ 67 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 68 FROM package 69 WHERE package.source_type = {0} AND 70 package.webhook_rebuild = {1} AND 71 package.source_json ILIKE {placeholder} 72 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url_subpart+'%' 73 ) 74 return [ScmPackage(row) for row in rows]
75
76 - def is_dir_in_commit(self, raw_commit_text):
77 if not self.subdirectory or not raw_commit_text: 78 return True 79 80 for line in raw_commit_text.split('\n'): 81 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/.*$', line) 82 if match and match.group(2).lower() == self.subdirectory.strip('/').lower(): 83 return True 84 85 return False
86
87 88 -def build_on_fedmsg_loop():
89 log.debug("Setting up poller...") 90 91 endpoint = 'tcp://hub.fedoraproject.org:9940' 92 topic = 'io.pagure.prod.pagure.git.receive' 93 94 ctx = zmq.Context() 95 s = ctx.socket(zmq.SUB) 96 s.connect(endpoint) 97 98 s.setsockopt(zmq.SUBSCRIBE, topic) 99 100 poller = zmq.Poller() 101 poller.register(s, zmq.POLLIN) 102 103 while True: 104 log.debug("Polling...") 105 evts = poller.poll(10000) 106 if not evts: 107 continue 108 109 log.debug("Receiving...") 110 topic, msg = s.recv_multipart() 111 112 log.debug("Parsing...") 113 data = json.loads(msg) 114 115 namespace = data['msg']['repo']['namespace'] 116 repo_name = data['msg']['repo']['name'] 117 branch = data['msg']['branch'] 118 start_commit = data['msg']['start_commit'] 119 end_commit = data['msg']['end_commit'] 120 121 if namespace: 122 clone_url_subpart = '/' + namespace + '/' + repo_name 123 else: 124 clone_url_subpart = '/' + repo_name 125 126 log.info("MSG:") 127 log.info("\tclone_url_subpart = {}".format(clone_url_subpart)) 128 log.info("\tbranch = {}".format(branch)) 129 130 candidates = ScmPackage.get_candidates_for_rebuild(clone_url_subpart) 131 132 raw_commit_text = None 133 # if start_commit != end_commit, then more than one commit and no means to iterate over 134 if candidates and start_commit == end_commit: 135 raw_commit_url = PAGURE_BASE_URL + clone_url_subpart + '/raw/' + start_commit 136 r = requests.get(raw_commit_url) 137 if r.status_code == requests.codes.ok: 138 raw_commit_text = r.text 139 else: 140 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url)) 141 142 for pkg in candidates: 143 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict)) 144 if PAGURE_HOSTNAME in urlparse(pkg.clone_url).netloc \ 145 and (pkg.clone_url.endswith(clone_url_subpart) or pkg.clone_url.endswith(clone_url_subpart+'.git')) \ 146 and (not pkg.committish or branch.endswith('/'+pkg.committish)) \ 147 and pkg.is_dir_in_commit(raw_commit_text): 148 log.info("\t -> rebuilding.") 149 pkg.build() 150 else: 151 log.info("\t -> skipping.")
152 153 154 if __name__ == '__main__': 155 while True: 156 try: 157 build_on_fedmsg_loop() 158 except: 159 log.exception("Error in fedmsg loop. Restarting it.") 160