diff -r d86a81540a71 -r b3d1b356e544 hgpoll.py --- a/hgpoll.py Sat Jul 18 16:59:44 2015 +0300 +++ b/hgpoll.py Sun Aug 02 17:15:00 2015 +0300 @@ -26,7 +26,6 @@ THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ''' -import hgapi import time import re import bt as Bt @@ -155,10 +154,9 @@ return None -' Check if a repository exists ' def check_repo_exists (repo_name): - print 'Checking that %s exists...' % repo_name - zanrepo = hgapi.Repo (repo_name) + ' Check if a repository exists ' + print ('Checking that %s exists...' % repo_name) if not os.path.exists (repo_name): os.makedirs (repo_name) @@ -168,9 +166,9 @@ repo_url = get_repo_info (repo_name).get_value ('url') try: print ('Cloning %s...' % repo_name) - zanrepo.hg_clone (repo_url, repo_name) + subprocess.call (['hg', 'clone', repo_url, repo_name]) - # We need to un-alias a few things, they can be aliased on the host machine (e.g. mine) + # We need to un-alias a few things, they may be aliased on the host machine (e.g. mine) comms=['log', 'incoming', 'pull', 'commit', 'push', 'outgoing', 'strip', 'transplant'] try: with open (os.path.join (repo_name, '.hg', 'hgrc'), 'a') as fp: @@ -182,66 +180,56 @@ print ('Unable to clone %s from %s: %s' % (repo_name, repo_url, e)) quit (1) - class HgProcessError (Exception): def __init__ (self, value): self.message = value def __str__ (self): return self.message +def contains_published_repositories (reponames): + for reponame in reponames: + if is_published (reponame): + return True + + return False + +def is_published (reponame): + repoinfo = get_repo_info (reponame) + return (repoinfo and not repoinfo.get_value ('extrarepo', default=False)) + def announce_ticket_resolved (ticket_id, cset): ticket_id = int (ticket_id) reponames = g_CommitsDb.get_commit_repos (cset) - if not reponames: - raise HgProcessError ('Changeset %s does not appear to exist!' % cset) - - for reponame in reponames: - repoinfo = get_repo_info (reponame) - if not repoinfo: - raise HgProcessError ('Unknown repo %s' % reponame) + if not contains_published_repositories (reponames): + raise HgProcessError ('Changeset %s is only committed to non-published repositories: %s' % + (cset, ', '.join (reponames))) - if not repoinfo.get_value ('extrarepo', default=False): - break - else: - raise HgProcessError ('Changeset %s is only committed to non-published repositories %s' % - (cset, reponames)) - - repo = hgapi.Repo (reponame) repo_url = repoinfo.get_value ('url', default=None) if not repo_url: raise HgProcessError ('Repo %s has no url!' % reponame) # Acquire additional data - moredata = get_commit_data (repo, cset, - r"{author|nonempty}\n{date(date, '%A %d %B %Y %H:%M:%S')}").split('\n') + moredata = get_commit_data (reponame, cset, + r"{author|person}\n{date(date, '%A %d %B %Y %H:%M:%S')}\n{author|email}").split('\n') if len (moredata) != 2: raise HgProcessError ('malformed hg data while processing %s' % cset) commit_author = moredata[0] commit_date = moredata[1] - commit_email = "" - commit_message = repo.hg_command ('log', '-r', cset, '--template', '{desc}') + commit_email = moredata[2] + commit_message = subprocess.check_output (['hg', '--cwd', reponame, 'log', '--rev', cset, '--template', '{desc}']) + commit_diffstat = subprocess.check_output (['hg', '--cwd', reponame, 'diff', '--change', cset, '--stat']) try: ticket_data = Bt.get_issue (ticket_id) except Exception as e: raise HgProcessError ("error while processing %s: %s" % (cset, e)) - # Remove the email address from the author if possible - rex = re.compile (r'^(.+) <([^>]+)>$.*') - match = rex.match (commit_author) - if match: - commit_author = match.group (1) - commit_email = match.group (2) - - commit_diffstat = repo.hg_command ('diff', '--change', cset, '--stat') - if len(commit_diffstat) > 0: commit_diffstat = 'Changes in files:\n[code]\n' + commit_diffstat + '\n[/code]' - # commit_diffstat = 'Changes in files:\n' + bbcodify(commit_diffstat) else: commit_diffstat = 'No changes in files.' @@ -282,7 +270,8 @@ for channel in irc_client.channels: if channel.get_value ('btannounce', default=True): irc_client.privmsg (channel.get_value ('name'), - "\003%d%s\003: commit\0035 %s\003 addresses issue\002\0032 %d\002" % (color_for_repo (reponame), reponame, cset, ticket_id)) + "\003%d%s\003: commit\0035 %s\003 addresses issue\002\0032 %d\002" % \ + (color_for_repo (reponame), reponame, cset, ticket_id)) irc_client.privmsg (channel.get_value ('name'), "Read all about it here: " + Bt.get_ticket_url (ticket_id)) @@ -298,59 +287,26 @@ global repocheck_timeout global g_CommitsDb - for repo in all_repo_names(): - check_repo_exists (repo) + for reponame in all_repo_names(): + check_repo_exists (reponame) g_CommitsDb = CommitsDb() repocheck_timeout = time.time() + 15 - global ZDoomRevNumber - try: - ZDoomRevNumber = get_zdrev_number ('zandronum-merge-experiments', 'tip') - except Exception as e: - print 'error while figuring out zdoom rev number: %s' % e - -def get_commit_data (repo, rev, template): - return repo.hg_command ('log', '-l', '1', '-r', rev, '--template', template) - -def decipher_hgapi_error (e): - # Blah, hgapi, why must your error messages be so mangled? - try: - rawmsg = e.message.replace('\n', '').replace('" +','').replace('\t','') - errmsg = re.compile (r'.*: tErr: (.*)Out:.*').match (rawmsg).group (1) - return [True, errmsg] - except: - return [False, ''] - -def bbcodify (commit_diffstat): - result = '' - - for line in commit_diffstat.split('\n'): - rex = re.compile (r'^(.*)\|(.*) (\+*)(-*)(.*)$') - match = rex.match (line) - if match: - line = '%s|%s [color=#5F7]%s[/color][color=#F53]%s[/color]%s\n' \ - % (match.group (1), match.group (2), match.group (3), - match.group (4), match.group (5)) - - # Tracker doesn't seem to like empty color tags - line = line.replace ('[color=#5F7][/color]', '').replace ('[color=#F53][/color]', '') - - result += line - - return result +def get_commit_data (reponame, rev, template): + return subprocess.check_output (['hg', '--cwd', reponame, 'log', '--limit', '1', '--rev', rev, '--template', template]) def poll(): global repocheck_timeout if time.time() < repocheck_timeout: return + hgns = Config.get_node ('hg') + repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60 + for reponame in all_repo_names(): poll_one_repo (reponame) - hgns = Config.get_node ('hg') - repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60 - def poll_one_repo (repo_name): global repocheck_timeout hgns = Config.get_node ('hg') @@ -358,35 +314,36 @@ if not hgns.get_value ('track', default=True): return - repo = hgapi.Repo (repo_name) commit_data = [] delimeter = '^^^^^^^^^^' delimeter2 = '@@@@@@@@@@' + maxcommits = 15 + numcommits = 0 print 'Checking %s for updates' % repo_name try: - data = repo.hg_command ('incoming', '--quiet', '--template', - delimeter.join (['{node|short}', '{desc}']) + delimeter2).split (delimeter2) - except hgapi.hgapi.HgException as e: - deciphered = decipher_hgapi_error (e) - - if deciphered[0] and len(deciphered[1]) > 0: - Irc.broadcast ("error while using hg incoming on %s: %s" % (repo_name, deciphered[1])) - + data = subprocess.check_output (['hg', '--cwd', repo_name, 'incoming', + '--limit', str(maxcommits), '--quiet', '--template', + delimeter.join (['{node|short}', '{desc}']) + delimeter2]).split (delimeter2) + except subprocess.CalledProcessError: return except Exception as e: - Irc.broadcast ("%s" % `e`) + Irc.broadcast (e.__class__.__name__ + ": " + str (e)) return - if not data: - print ('No updates to %s' % repo_name) - for line in data: if line: + numcommits += 1 commit_data.append (line.split (delimeter)) process_new_commits (repo_name, commit_data) + if numcommits == maxcommits: + # If we have 25 commits here, there may be more coming so recheck sooner + global repocheck_timeout + print ('Processed %d commits, checking for new commits in 1 minute...' % len(data)) + repocheck_timeout = time.time() + 60 + def process_new_commits (repo_name, commit_data): if len (commit_data) == 0: return @@ -394,73 +351,80 @@ repo_name = repo_name.lower() repo_url = get_repo_info (repo_name).get_value ('url') isExtraRepo = get_repo_info (repo_name).get_value ('extrarepo', False) - zanrepo = hgapi.Repo (repo_name) - print '%d new commits on %s' % (len (commit_data), repo_name) + print ('%d new commits on %s' % (len (commit_data), repo_name)) pull_args = [] messages = [[], [], []] - messageSizeClass = 2 for commit in commit_data: pull_args.append ('-r'); pull_args.append (commit[0]); - print 'Pulling new commits...' + print ('Pulling new commits...') try: - zanrepo.hg_command ('pull', *pull_args) + subprocess.call (['hg', '--cwd', repo_name, 'pull'] + pull_args) except Exception as e: Irc.broadcast ('Warning: unable to pull: %s' % `e`) return + LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL = range (0, 3) + for commit in commit_data: commit_node = commit[0] commit_message = commit[1] - print 'Processing new commit %s...' % commit_node + print ('Processing new commit %s...' % commit_node) try: existingrepos = g_CommitsDb.get_commit_repos (commit_node) alreadyAdded = len (existingrepos) > 0 delim = '@@@@@@@@@@' - data = get_commit_data (zanrepo, commit_node, delim.join (['{node}', '{author}', '{bookmarks}', \ - '{date|hgdate}'])).split (delim) + data = get_commit_data (repo_name, commit_node, delim.join (['{node}', '{author|person}', + '{bookmarks}', '{date|hgdate}', '{author|email}'])).split (delim) commit_full_node = data[0] commit_author = data[1] commit_bookmarks = prettify_bookmarks (data[2]) commit_time = int (data[3].split (' ')[0]) commit_url = '%s/commits/%s' % (repo_url, commit_node) - commit_email = '' + commit_email = data[4] + isMergeFromSandbox = False # If the commit was already in the commits database, it is not a new one and we should # not react to it. Still add it to the db though so that the new repo name is added. g_CommitsDb.add_commit (repo=repo_name, changeset=commit_full_node, timestamp=commit_time) + if alreadyAdded: - print ('''I already know of %s - they're in %s - not announcing.''' % - (commit_node, existingrepos)) - continue - - # Remove the email address from the author if possible - rex = re.compile (r'^(.+) <([^>]+)>$.*') - match = rex.match (commit_author) - if match: - commit_author = match.group (1) - commit_email = match.group (2) + if not contains_published_repositories (existingrepos) and is_published (repo_name): + isMergeFromSandbox = True + print ('''%s appears to be a merge from sandbox (exists in %s)''' % + (commit_node, existingrepos)) + else: + print ('''I already know of %s - they're in %s - not announcing.''' % + (commit_node, existingrepos)) + continue commit_trackeruser = Config.find_developer_by_email (commit_email) - committer = commit_trackeruser if commit_trackeruser != '' else commit_author - - commitDescriptor = """commit""" if int (math.ceil (random.random() * 100)) != 1 else """KERMIT""" + committer = commit_trackeruser if commit_trackeruser else commit_author + commitDescriptor = """commit""" if int (random.random() * 100) != 0 else """KERMIT""" - commitMessage = """\003%d%s\003: new %s\0035 %s%s\003 by\0032 %s\003: %s""" % \ - (color_for_repo (repo_name), repo_name, commitDescriptor, commit_node, commit_bookmarks, - committer, utility.shorten_link (commit_url)) + if not isMergeFromSandbox: + commitMessage = """\003%d%s\003: new %s\0035 %s%s\003 by\0032 %s\003: %s""" % \ + (color_for_repo (repo_name), repo_name, commitDescriptor, commit_node, commit_bookmarks, + committer, utility.shorten_link (commit_url)) + + for length in [LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL]: + messages[length].append (commitMessage) - messages[0].append (commitMessage) - messages[1].append (commitMessage) - messages[2].append (commitMessage) - messages[1].append (' ' + commit_message.splitlines()[0]) + messages[LENGTH_SHORT].append (' ' + commit_message.splitlines()[0]) - for line in commit_message.splitlines()[0:4]: - messages[2].append (' ' + line) + for line in commit_message.splitlines()[0:4]: + messages[LENGTH_FULL].append (' ' + line) + else: + commitMessage = """\003%d%s\003: %s\0035 %s%s\003 by\0032 %s\003 was pulled: %s""" % \ + (color_for_repo (repo_name), repo_name, commitDescriptor, commit_node, commit_bookmarks, + committer, utility.shorten_link (commit_url)) + + for length in [LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL]: + messages[length].append (commitMessage) if not isExtraRepo: rex = re.compile (r'^.*(fixes|resolves|addresses|should fix) ([0-9]+).*$') @@ -478,12 +442,11 @@ fullMessageLength = len (''.join (messages[2])) if fullMessageLength > 3000: - messageSizeClass = 0 + messageSizeClass = LENGTH_MINIMUM elif fullMessageLength > 768: - messageSizeClass = 1 - - print ("""Message length in total: %d, using size class %d (%d)""" %\ - (fullMessageLength, messageSizeClass, len (''.join (messages[messageSizeClass])))) + messageSizeClass = LENGTH_SHORT + else: + messageSizeClass = LENGTH_FULL # Post it all on IRC now for message in messages[messageSizeClass]: @@ -497,10 +460,6 @@ irc_client.privmsg (channel.get_value ('name'), message) - # Hack for ZDoom upgrades - if repo_name == 'zandronum-merge-experiments': - check_zdoom_upgrade (repo_name, commit_node) - g_CommitsDb.commit() def make_progress_bar (p, barLength, colored=True): @@ -510,53 +469,6 @@ + ColorChar + '1,1' + ('-' * int (barLength - round (p * barLength))) \ + ColorChar + BoldChar + ']' + BoldChar -ZDoomMin = 2560 -ZDoomMax = 4341 - -def check_zdoom_upgrade (repo_name, commit_node): - zanrepo = hgapi.Repo (repo_name) - try: - global ZDoomRevNumber - newnumber = get_zdrev_number (repo_name, commit_node) - - if newnumber > ZDoomRevNumber: - ZDoomRevNumber = newnumber - update_zadev_topic() - except Exception as e: - Irc.broadcast ('Error while dealing with ZDoom upgrade number: %s' % e) - -def update_zadev_topic(): - return - fraction = float (ZDoomRevNumber - ZDoomMin) / (ZDoomMax - ZDoomMin) - topicText = """ZDoom r%d progress: at r%d, %d revisions left (%d%% complete)""" %\ - (ZDoomMax, ZDoomRevNumber, ZDoomMax - ZDoomRevNumber, round (fraction * 100)) - - try: - Irc.ClientsByName['zandronum'].write ("""TOPIC #zadev :%s""" % topicText) - Irc.ClientsByName['zandronum'].write ("""TOPIC #commits :%s""" % topicText) - except Exception as e: - Irc.broadcast ("""Error setting #zadev topic: %s""" % e) - -def get_zdrev_number (repo_name, commit_node): - zanrepo = hgapi.Repo (repo_name) - subprocess.call (['hg', '--cwd', repo_name, 'revert', '-r', commit_node, 'src/version.h']) - rx = re.compile (r'#define\s+ZD_SVN_REVISION_NUMBER\s+([0-9]+)') - result = None - - with open (repo_name + '/src/version.h') as version_file: - for line in version_file: - match = rx.match (line) - if match != None: - result = int (match.group (1)) - break - - subprocess.call (['hg', '--cwd', repo_name, 'revert', '-r.', 'src/version.h']) - - if result != None: - return result - - raise ValueError ("""Could not find ZD_SVN_REVISION_NUMBER in version.h""") - def force_poll(): global repocheck_timeout repocheck_timeout = 0