hgpoll.py

import hgapi
import time
import re
import bt as Bt
import irc as Irc
import os, sqlite3, subprocess
from datetime import datetime
from configfile import Config
import utility
g_needCommitsTxtRebuild = True

class CommitsDb (object):
	def __init__(self):
		needNew = not os.path.isfile ('commits.db')
		self.db = sqlite3.connect ('commits.db')

		if needNew:
			self.createNew()

	def create_new (self):
		self.db.executescript ('''
			DROP TABLE IF EXISTS COMMITS;
			DROP TABLE IF EXISTS REPOS;
			DROP TABLE IF EXISTS REPOCOMMITS;
			CREATE TABLE IF NOT EXISTS COMMITS
			(
				Node        text NOT NULL,
				Dateversion text NOT NULL,
				PRIMARY KEY (Node)
			);
			CREATE TABLE IF NOT EXISTS REPOS
			(
				Name     text NOT NULL,
				PRIMARY KEY (Name)
			);
			CREATE TABLE IF NOT EXISTS REPOCOMMITS
			(
				Reponame text,
				Node     text,
				FOREIGN KEY (Reponame) REFERENCES REPOS(Name),
				FOREIGN KEY (Node) REFERENCES COMMITS(Node)
			);
		''')

		print 'Building commits.db...'
		for repo in all_repo_names():
			print 'Adding commits from %s...' % repo

			data = subprocess.check_output (['hg', '--cwd', repo, 'log', '--template',
				'{node} {date|hgdate}\n']).splitlines()

			for line in data:
				changeset, timestamp, tz = line.split(' ')
				self.add_commit (repo, changeset, int (timestamp))

		self.commit()

	def add_commit (self, repo, changeset, timestamp):
		dateversion = datetime.utcfromtimestamp (timestamp).strftime ('%y%m%d-%H%M')
		self.db.execute ('''
			INSERT OR IGNORE INTO REPOS
			VALUES (?)
		''', (repo,))

		self.db.execute ('''
			INSERT OR IGNORE INTO COMMITS
			VALUES (?, ?)
		''', (changeset, dateversion))

		self.db.execute ('''
			INSERT INTO REPOCOMMITS 
			VALUES (?, ?)
		''', (repo, changeset))

	def get_commit_repos (self, node):
		cursor = self.db.execute ('''
			SELECT Reponame
			FROM REPOCOMMITS
			WHERE Node LIKE ?
		''', (node + '%',))

		results = cursor.fetchall()
		return list (set (zip (*results)[0])) if results else []

	def find_commit_by_dateversion (self, dateversion):
		cursor = self.db.execute ('''
			SELECT Node
			FROM COMMITS
			WHERE Dateversion = ?
		''', (dateversion,))
		result = cursor.fetchone()
		return result[0] if result else None

	def commit(self):
		self.db.commit()

g_CommitsDb = CommitsDb()

def color_for_repo (repo_name):
	repo_name = repo_name.lower()
	repoconfig = Config.get_node ('hg').get_node ('repos')

	if repoconfig.has_node (repo_name):
		return repoconfig.get_node(repo_name).get_value ('colorcode', 0)

	return 0

def prettify_bookmarks (bookmarks):
	return "\0036 [\002%s\002]" % bookmarks

def get_repo_info (reponame):
	reponame = reponame.lower()
	repoconfig = Config.get_node ('hg').get_node ('repos')

	if repoconfig.has_node (reponame):
		return repoconfig.get_node (reponame)

	return None

' Check if a repository exists '
def check_repo_exists (repo_name):
	print 'Checking that %s exists...' % repo_name
	zanrepo = hgapi.Repo (repo_name)

	if not os.path.exists (repo_name):
		os.makedirs (repo_name)

	try:
		zanrepo.hg_command ('id', '.')
	except hgapi.hgapi.HgException:
		# If the repo does not exist, clone it.
		repo_url = get_repo_info (repo_name).get_value ('url')
		try:
			print ('Cloning %s...' % repo_name)
			zanrepo.hg_clone (repo_url, repo_name)
			print ('Cloning done.')
		except Exception as e:
			print ('Unable to clone %s from %s: %s' % (repo_name, repo_url, e))
			quit (1)


class HgProcessError (Exception):
	def __init__ (self, value):
		self.message = value
	def __str__ (self):
		return self.message

def announce_ticket_resolved (ticket_id, cset):
	ticket_id = int (ticket_id)
	reponames = g_CommitsDb.get_commit_repos (cset)

	if not reponames:
		raise HgProcessError ('Changeset %s does not appear to exist!' % cset)

	for reponame in reponames:
		repoinfo = get_repo_info (reponame)
		if not repoinfo:
			raise HgProcessError ('Unknown repo %s' % reponame)

		if not repoinfo.get_value ('private', default=False):
			break
	else:
		raise HgProcessError ('Changeset %s is only committed to non-published repositories %s' %
			(cset, reponames))

	repo = hgapi.Repo (reponame)
	repo_url = repoinfo.get_value ('url', default=None)

	if not repo_url:
		raise HgProcessError ('Repo %s has no url!' % reponame)

	# Acquire additional data
	moredata = get_commit_data (repo, cset,
		r"{author|nonempty}\n{date(date, '%A %d %B %Y %H:%M:%S')}").split('\n')

	if len (moredata) != 2:
		raise HgProcessError ('malformed hg data while processing %s' % cset)

	commit_author = moredata[0]
	commit_date = moredata[1]
	commit_email = ""
	commit_message = repo.hg_command ('log', '-r', cset, '--template', '{desc}')

	try:
		ticket_data = Bt.get_issue (ticket_id)
	except Exception as e:
		raise HgProcessError ("error while processing %s: %s" % (cset, e))

	# Remove the email address from the author if possible
	rex = re.compile (r'^(.+) <([^>]+)>$.*')
	match = rex.match (commit_author)
	if match:
		commit_author = match.group (1)
		commit_email = match.group (2)

	commit_diffstat = repo.hg_command ('diff', '--change', cset, '--stat')

	if len(commit_diffstat) > 0:
		# commit_diffstat = 'Changes in files:\n[code]\n' + commit_diffstat + '\n[/code]'
		commit_diffstat = 'Changes in files:\n' + bbcodify(commit_diffstat)
	else:
		commit_diffstat = 'No changes in files.'

	# Compare the email addresses against known developer usernames
	commit_trackeruser = Config.find_developer_by_email (commit_email)

	if commit_trackeruser != '':
		commit_author += ' [%s]' % commit_trackeruser

	commit_message = commit_message.replace ("\n", " ")

	message = 'Issue addressed by commit %s: [b][url=%s/commits/%s]%s[/url][/b]' \
		% (cset, repo_url, cset, commit_message)
	message += "\nCommitted by %s on %s\n\n%s" \
			% (commit_author, commit_date, commit_diffstat)

	need_update = False

	# If not already set, set handler
	if not 'handler' in ticket_data:
		ticket_data['handler'] = {'name': commit_trackeruser}
		need_update = True

	# Find out the status level of the ticket
	needs_testing_level = 70

	if ticket_data['status']['id'] < needs_testing_level:
		ticket_data.status['id'] = needs_testing_level
		need_update = True

	# Set target version if not set
	if not 'target_version' in ticket_data:
		ticket_data['target_version'] = repoinfo.get_value ('version')
		need_update = True

	# Announce on IRC
	for irc_client in Irc.all_clients:
		for channel in irc_client.channels:
			if channel.get_value ('btannounce', default=True):
				irc_client.privmsg (channel.get_value ('name'),
					"\003%d%s\003: commit\0035 %s\003 addresses issue\002\0032 %d\002" % (color_for_repo (reponame), reponame, cset, ticket_id))
				irc_client.privmsg (channel.get_value ('name'),
					"Read all about it here: " + Bt.get_ticket_url (ticket_id))

	if need_update:
		# We need to remove the note data, otherwise the ticket notes
		# will get unnecessary updates. WTF, MantisBT?
		ticket_data.notes = []
		Bt.update_issue (ticket_id, ticket_data)

	Bt.post_note (ticket_id, message)

def all_repo_names():
	return Config.get_node ('hg').get_value ('repos', {}).keys()

def init():
	for repo in all_repo_names():
		check_repo_exists (repo)
	global repocheck_timeout
	repocheck_timeout = time.time() + 15

def get_commit_data (repo, rev, template):
	return repo.hg_command ('log', '-l', '1', '-r', rev, '--template', template)

def decipher_hgapi_error (e):
	# Blah, hgapi, why must your error messages be so mangled?
	try:
		rawmsg = e.message.replace('\n', '').replace('" +','').replace('\t','')
		errmsg = re.compile (r'.*: tErr: (.*)Out:.*').match (rawmsg).group (1)
		return [True, errmsg]
	except:
		return [False, '']

def bbcodify (commit_diffstat):
	result = ''

	for line in commit_diffstat.split('\n'):
		rex = re.compile (r'^(.*)\|(.*) (\+*)(-*)(.*)$')
		match = rex.match (line)
		if match:
			line = '%s|%s [color=#5F7]%s[/color][color=#F53]%s[/color]%s\n' \
				% (match.group (1), match.group (2), match.group (3),
				match.group (4), match.group (5))

			# Tracker doesn't seem to like empty color tags
			line = line.replace ('[color=#5F7][/color]', '').replace ('[color=#F53][/color]', '')

		result += line

	return result

def poll():
	global repocheck_timeout
	if time.time() < repocheck_timeout:
		return

	for reponame in all_repo_names():
		poll_one_repo (reponame)

	hgns = Config.get_node ('hg')
	repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60

def poll_one_repo (repo_name):
	global repocheck_timeout
	hgns = Config.get_node ('hg')

	if not hgns.get_value ('track', default=True):
		return

	repo = hgapi.Repo (repo_name)
	commit_data = []
	delimeter = '@@@@@@@@@@'
	print 'Checking %s for updates' % repo_name

	try:
		data = repo.hg_command ('incoming', '--quiet', '--template',
			'{node|short} {desc}' + delimeter).split (delimeter)
	except hgapi.hgapi.HgException as e:
		deciphered = decipher_hgapi_error (e)

		if deciphered[0] and len(deciphered[1]) > 0:
			Irc.broadcast ("error while using hg import on %s: %s" % (repo_name, deciphered[1]))

		return
	except Exception as e:
		Irc.broadcast ("%s" % `e`)
		return

	if not data:
		print ('No updates to %s' % repo_name)

	for line in data:
		if line == '':
			continue

		rex = re.compile (r'([^ ]+) (.+)')
		match = rex.match (line)
		failed = False

		if not match:
			Irc.broadcast ('malformed hg data: %s' % line)
			continue

		commit_node = match.group (1)
		commit_message = match.group (2)
		commit_data.append ([commit_node, commit_message])

	process_new_commits (repo_name, commit_data)

def process_new_commits (repo_name, commit_data):
	if len (commit_data) == 0:
		return

	repo_name = repo_name.lower()
	repo_url = get_repo_info (repo_name).get_value ('url')
	ispublishing = not get_repo_info (repo_name).get_value ('private', False)
	num_commits = 0
	zanrepo = hgapi.Repo (repo_name)
	print '%d new commits on %s' % (len (commit_data), repo_name)

	# Drop commits that we already have
	i = 0
	while i < len (commit_data):
		try:
			zanrepo.hg_command ('log', '-r', commit_data[i][0])
			del commit_data[i]
		except:
			i += 1

	pull_args = [];

	for commit in commit_data:
		pull_args.append ('-r');
		pull_args.append (commit[0]);

	print 'Pulling new commits...'
	try:
		zanrepo.hg_command ('pull', *pull_args)
	except Exception as e:
		Irc.broadcast ('Warning: unable to pull: %s' % `e`)
		return

	for commit in commit_data:
		commit_node = commit[0]
		commit_message = commit[1]
		print 'Processing new commit %s...' % commit_node

		try:
			alreadyAdded = len (g_CommitsDb.get_commit_repos (commit_node)) > 0

			delim = '@@@@@@@@@@'
			data = get_commit_data (zanrepo, commit_node, delim.join (['{author}', '{bookmarks}', \
				'{date|hgdate}'])).split (delim)
			print 'data is: %s' % data
			commit_author = data[0]
			commit_bookmarks = prettify_bookmarks (data[1])
			commit_time = int (data[2].split (' ')[0])
			commit_url = '%s/commits/%s' % (repo_url, commit_node)
			commit_email = ''

			# If the commit was already in the commits database, it is not a new one and we should
			# not react to it. Still add it to the db though so that the new repo name is added.
			g_CommitsDb.add_commit (repo=repo_name, changeset=commit_node, timestamp=commit_time)
			if alreadyAdded:
				continue

			# Remove the email address from the author if possible
			rex = re.compile (r'^(.+) <([^>]+)>$.*')
			match = rex.match (commit_author)
			if match:
				commit_author = match.group (1)
				commit_email = match.group (2)

			commit_trackeruser = Config.find_developer_by_email (commit_email)
			committer = commit_trackeruser if commit_trackeruser != '' else commit_author

			for irc_client in Irc.all_clients:
				for channel in irc_client.channels:
					if not channel.get_value ('btannounce', False):
						continue

					if ispublishing or channel.get_value ('btprivate', False):
						irc_client.privmsg (channel.get_value ('name'),
							"\003%d%s\003: new commit\0035 %s%s\003 by\0032 %s\003: %s"
							% (color_for_repo (repo_name), repo_name, commit_node, commit_bookmarks,
							committer, utility.shorten_link (commit_url)))

						for line in commit_message.split ('\n'):
							irc_client.privmsg (channel.get_value ('name'), line)

			if ispublishing:
				rex = re.compile (r'^.*(fixes|resolves|addresses|should fix) ([0-9]+).*$')
				match = rex.match (commit_message)

				if match:
					announce_ticket_resolved (match.group (2), commit_node)

			num_commits += 1
		except Exception as e:
			Irc.broadcast ('Error while processing %s: %s' % (commit_node, e))
			continue

	g_CommitsDb.commit()

def force_poll():
	global repocheck_timeout
	repocheck_timeout = 0
	poll()