Package run :: Module build_on_pagure_commit
[hide private]
[frames] | no frames]

Source Code for Module run.build_on_pagure_commit

  1  #!/usr/bin/env python 
  2   
  3  import json 
  4  import pprint 
  5  import zmq 
  6  import sys 
  7  import os 
  8  import logging 
  9  import requests 
 10  import re 
 11   
 12  sys.path.append( 
 13      os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
 14  ) 
 15   
 16  from coprs import db, app 
 17  from coprs.logic.coprs_logic import CoprsLogic 
 18  from coprs.logic.packages_logic import PackagesLogic 
 19  from coprs.logic.complex_logic import ComplexLogic 
 20  from coprs import helpers 
 21   
 22  try: 
 23      from urllib.parse import urlparse 
 24  except ImportError: 
 25      from urlparse import urlparse 
 26   
 27  SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm") 
 28   
 29  logging.basicConfig( 
 30      filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")), 
 31      format='[%(asctime)s][%(levelname)6s]: %(message)s', 
 32      level=logging.DEBUG) 
 33   
 34  log = logging.getLogger(__name__) 
 35  log.addHandler(logging.StreamHandler(sys.stdout)) 
 36   
 37  PAGURE_BASE_URL = "https://pagure.io/" 
 38  PAGURE_HOSTNAME = "pagure.io" 
 39   
 40  ENDPOINT = 'tcp://hub.fedoraproject.org:9940' 
 41  TOPIC = 'io.pagure.prod.pagure.git.receive' 
42 43 44 -class ScmPackage(object):
45 - def __init__(self, db_row):
46 self.pkg_id = db_row.package_id 47 self.copr_id = db_row.copr_id 48 49 self.source_json_dict = json.loads(db_row.source_json) 50 self.clone_url = self.source_json_dict.get('clone_url') or '' 51 self.committish = self.source_json_dict.get('committish') or '' 52 self.subdirectory = self.source_json_dict.get('subdirectory') or '' 53 54 self.copr = CoprsLogic.get_by_id(self.copr_id).first() 55 self.package = ComplexLogic.get_package_by_id_safe(self.pkg_id)
56
57 - def build(self):
58 PackagesLogic.build_package(self.copr.user, self.copr, self.package) 59 db.session.commit()
60 61 @classmethod
62 - def get_candidates_for_rebuild(cls, clone_url):
63 if db.engine.url.drivername == "sqlite": 64 placeholder = '?' 65 true = '1' 66 else: 67 placeholder = '%s' 68 true = 'true' 69 70 rows = db.engine.execute( 71 """ 72 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id 73 FROM package 74 WHERE package.source_type = {0} AND 75 package.webhook_rebuild = {1} AND 76 package.source_json ILIKE {placeholder} 77 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url+'%' 78 ) 79 return [ScmPackage(row) for row in rows]
80
81 - def is_dir_in_commit(self, raw_commit_text):
82 if not self.subdirectory or not raw_commit_text: 83 return True 84 85 for line in raw_commit_text.split('\n'): 86 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line) 87 if match and match.group(2).lower() == self.subdirectory.strip('./').lower(): 88 return True 89 90 return False
91
92 93 -def build_on_fedmsg_loop():
94 log.debug("Setting up poller...") 95 96 ctx = zmq.Context() 97 s = ctx.socket(zmq.SUB) 98 s.connect(ENDPOINT) 99 100 s.setsockopt(zmq.SUBSCRIBE, TOPIC) 101 102 poller = zmq.Poller() 103 poller.register(s, zmq.POLLIN) 104 105 while True: 106 log.debug("Polling...") 107 evts = poller.poll(10000) 108 if not evts: 109 continue 110 111 log.debug("Receiving...") 112 topic, msg = s.recv_multipart() 113 114 log.debug("Parsing...") 115 data = json.loads(msg) 116 117 project_url_path = data['msg']['repo']['url_path'] 118 clone_url_path = data['msg']['repo']['fullname'] 119 branch = data['msg']['branch'] 120 start_commit = data['msg']['start_commit'] 121 end_commit = data['msg']['end_commit'] 122 clone_url = PAGURE_BASE_URL + clone_url_path 123 124 log.info("MSG:") 125 log.info("\tclone_url_path = {}".format(clone_url_path)) 126 log.info("\tbranch = {}".format(branch)) 127 128 candidates = ScmPackage.get_candidates_for_rebuild(clone_url) 129 raw_commit_text = None 130 if candidates: 131 # if start_commit != end_commit 132 # then more than one commit and no means to iterate over :( 133 raw_commit_url = PAGURE_BASE_URL + project_url_path + '/raw/' + end_commit 134 r = requests.get(raw_commit_url) 135 if r.status_code == requests.codes.ok: 136 raw_commit_text = r.text 137 else: 138 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url)) 139 140 for pkg in candidates: 141 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict)) 142 if (pkg.clone_url == clone_url or pkg.clone_url == clone_url+'.git') \ 143 and (not pkg.committish or branch.endswith('/'+pkg.committish)) \ 144 and pkg.is_dir_in_commit(raw_commit_text): 145 log.info("\t -> rebuilding.") 146 pkg.build() 147 else: 148 log.info("\t -> skipping.")
149 150 151 if __name__ == '__main__': 152 while True: 153 try: 154 build_on_fedmsg_loop() 155 except KeyboardInterrupt: 156 sys.exit(1) 157 except: 158 log.exception('Error in fedmsg loop. Restarting it.') 159