hgpoll.py

'''
	Copyright 2014-2015 Teemu Piippo
	All rights reserved.

	Redistribution and use in source and binary forms, with or without
	modification, are permitted provided that the following conditions
	are met:

	1. Redistributions of source code must retain the above copyright
	   notice, this list of conditions and the following disclaimer.
	2. Redistributions in binary form must reproduce the above copyright
	   notice, this list of conditions and the following disclaimer in the
	   documentation and/or other materials provided with the distribution.
	3. The name of the author may not be used to endorse or promote products
	   derived from this software without specific prior written permission.

	THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

import hgapi
import time
import re
import bt as Bt
import irc as Irc
import os, sqlite3, subprocess
from datetime import datetime
from configfile import Config
import utility
import subprocess
import random
import math

g_CommitsDb = None
ZDoomRevNumber = 0

def all_repo_names():
	return Config.get_node ('hg').get_value ('repos', {}).keys()

class CommitsDb (object):
	def __init__(self):
		dbname = Config.get_node('hg').get_value('commits_db', 'commits.db')
		needNew = not os.path.isfile (dbname)
		self.db = sqlite3.connect (dbname)

		if needNew:
			self.create_new()

	def create_new (self):
		self.db.executescript ('''
			DROP TABLE IF EXISTS COMMITS;
			DROP TABLE IF EXISTS REPOS;
			DROP TABLE IF EXISTS REPOCOMMITS;
			CREATE TABLE IF NOT EXISTS COMMITS
			(
				Node        text NOT NULL,
				Dateversion text NOT NULL,
				PRIMARY KEY (Node)
			);
			CREATE TABLE IF NOT EXISTS REPOS
			(
				Name     text NOT NULL,
				PRIMARY KEY (Name)
			);
			CREATE TABLE IF NOT EXISTS REPOCOMMITS
			(
				Reponame text,
				Node     text,
				FOREIGN KEY (Reponame) REFERENCES REPOS(Name),
				FOREIGN KEY (Node) REFERENCES COMMITS(Node)
			);
		''')

		print 'Building commits.db...'
		for repo in all_repo_names():
			print 'Adding commits from %s...' % repo

			data = subprocess.check_output (['hg', '--cwd', repo, 'log', '--template',
				'{node} {date|hgdate}\n']).splitlines()

			for line in data:
				changeset, timestamp, tz = line.split(' ')
				self.add_commit (repo, changeset, int (timestamp))

		self.commit()

	def add_commit (self, repo, changeset, timestamp):
		dateversion = datetime.utcfromtimestamp (timestamp).strftime ('%y%m%d-%H%M')
		self.db.execute ('''
			INSERT OR IGNORE INTO REPOS
			VALUES (?)
		''', (repo,))

		self.db.execute ('''
			INSERT OR IGNORE INTO COMMITS
			VALUES (?, ?)
		''', (changeset, dateversion))

		self.db.execute ('''
			INSERT INTO REPOCOMMITS 
			VALUES (?, ?)
		''', (repo, changeset))

	def get_commit_repos (self, node):
		cursor = self.db.execute ('''
			SELECT Reponame
			FROM REPOCOMMITS
			WHERE Node LIKE ?
		''', (node + '%',))

		results = cursor.fetchall()
		return list (set (zip (*results)[0])) if results else []

	def find_commit_by_dateversion (self, dateversion):
		cursor = self.db.execute ('''
			SELECT Node
			FROM COMMITS
			WHERE Dateversion = ?
		''', (dateversion,))
		result = cursor.fetchone()
		return result[0] if result else None

	def commit(self):
		self.db.commit()

def color_for_repo (repo_name):
	repo_name = repo_name.lower()
	repoconfig = Config.get_node ('hg').get_node ('repos')

	if repoconfig.has_node (repo_name):
		return repoconfig.get_node(repo_name).get_value ('colorcode', 0)

	return 0

def prettify_bookmarks (bookmarks):
	if bookmarks:
		return "\0036 [\002%s\002]" % bookmarks
	else:
		return ''

def get_repo_info (reponame):
	reponame = reponame.lower()
	repoconfig = Config.get_node ('hg').get_node ('repos')

	if repoconfig.has_node (reponame):
		return repoconfig.get_node (reponame)

	return None

' Check if a repository exists '
def check_repo_exists (repo_name):
	print 'Checking that %s exists...' % repo_name
	zanrepo = hgapi.Repo (repo_name)

	if not os.path.exists (repo_name):
		os.makedirs (repo_name)

	if not os.path.isfile (os.path.join (repo_name, '.hg', 'hgrc')):
		# If the repo does not exist, clone it.
		repo_url = get_repo_info (repo_name).get_value ('url')
		try:
			print ('Cloning %s...' % repo_name)
			zanrepo.hg_clone (repo_url, repo_name)

			# We need to un-alias a few things, they can be aliased on the host machine (e.g. mine)
			comms=['log', 'incoming', 'pull', 'commit', 'push', 'outgoing', 'strip', 'transplant']
			try:
				with open (os.path.join (repo_name, '.hg', 'hgrc'), 'a') as fp:
					fp.write ('\n[alias]\n' + ''.join(['%s=%s\n' % (x, x) for x in comms]))
			except Exception as e:
				print ('Warning: unable to alter hgrc of %s: %s' % repo_name, e)
			print ('Cloning done.')
		except Exception as e:
			print ('Unable to clone %s from %s: %s' % (repo_name, repo_url, e))
			quit (1)


class HgProcessError (Exception):
	def __init__ (self, value):
		self.message = value
	def __str__ (self):
		return self.message

def announce_ticket_resolved (ticket_id, cset):
	ticket_id = int (ticket_id)
	reponames = g_CommitsDb.get_commit_repos (cset)

	if not reponames:
		raise HgProcessError ('Changeset %s does not appear to exist!' % cset)

	for reponame in reponames:
		repoinfo = get_repo_info (reponame)
		if not repoinfo:
			raise HgProcessError ('Unknown repo %s' % reponame)

		if not repoinfo.get_value ('extrarepo', default=False):
			break
	else:
		raise HgProcessError ('Changeset %s is only committed to non-published repositories %s' %
			(cset, reponames))

	repo = hgapi.Repo (reponame)
	repo_url = repoinfo.get_value ('url', default=None)

	if not repo_url:
		raise HgProcessError ('Repo %s has no url!' % reponame)

	# Acquire additional data
	moredata = get_commit_data (repo, cset,
		r"{author|nonempty}\n{date(date, '%A %d %B %Y %H:%M:%S')}").split('\n')

	if len (moredata) != 2:
		raise HgProcessError ('malformed hg data while processing %s' % cset)

	commit_author = moredata[0]
	commit_date = moredata[1]
	commit_email = ""
	commit_message = repo.hg_command ('log', '-r', cset, '--template', '{desc}')

	try:
		ticket_data = Bt.get_issue (ticket_id)
	except Exception as e:
		raise HgProcessError ("error while processing %s: %s" % (cset, e))

	# Remove the email address from the author if possible
	rex = re.compile (r'^(.+) <([^>]+)>$.*')
	match = rex.match (commit_author)
	if match:
		commit_author = match.group (1)
		commit_email = match.group (2)

	commit_diffstat = repo.hg_command ('diff', '--change', cset, '--stat')

	if len(commit_diffstat) > 0:
		# commit_diffstat = 'Changes in files:\n[code]\n' + commit_diffstat + '\n[/code]'
		commit_diffstat = 'Changes in files:\n' + bbcodify(commit_diffstat)
	else:
		commit_diffstat = 'No changes in files.'

	# Compare the email addresses against known developer usernames
	commit_trackeruser = Config.find_developer_by_email (commit_email)

	if commit_trackeruser != '':
		commit_author += ' [%s]' % commit_trackeruser

	commit_message = commit_message.replace ("\n", " ")

	message = 'Issue addressed by commit %s: [b][url=%s/commits/%s]%s[/url][/b]' \
		% (cset, repo_url, cset, commit_message)
	message += "\nCommitted by %s on %s\n\n%s" \
			% (commit_author, commit_date, commit_diffstat)

	need_update = False

	# If not already set, set handler
	if not 'handler' in ticket_data:
		ticket_data['handler'] = {'name': commit_trackeruser}
		need_update = True

	# Find out the status level of the ticket
	needs_testing_level = 70

	if ticket_data['status']['id'] < needs_testing_level:
		ticket_data.status['id'] = needs_testing_level
		need_update = True

	# Set target version if not set
	if not 'target_version' in ticket_data:
		ticket_data['target_version'] = repoinfo.get_value ('version')
		need_update = True

	# Announce on IRC
	for irc_client in Irc.all_clients:
		for channel in irc_client.channels:
			if channel.get_value ('btannounce', default=True):
				irc_client.privmsg (channel.get_value ('name'),
					"\003%d%s\003: commit\0035 %s\003 addresses issue\002\0032 %d\002" % (color_for_repo (reponame), reponame, cset, ticket_id))
				irc_client.privmsg (channel.get_value ('name'),
					"Read all about it here: " + Bt.get_ticket_url (ticket_id))

	if need_update:
		# We need to remove the note data, otherwise the ticket notes
		# will get unnecessary updates. WTF, MantisBT?
		ticket_data.notes = []
		Bt.update_issue (ticket_id, ticket_data)

	Bt.post_note (ticket_id, message)

def init():
	global repocheck_timeout
	global g_CommitsDb

	for repo in all_repo_names():
		check_repo_exists (repo)

	repocheck_timeout = time.time() + 15
	g_CommitsDb = CommitsDb()

	global ZDoomRevNumber
	try:
		ZDoomRevNumber = get_zdrev_number ('zandronum-merge-experiments', 'tip')
	except Exception as e:
		print 'error while figuring out zdoom rev number: %s' % e

def get_commit_data (repo, rev, template):
	return repo.hg_command ('log', '-l', '1', '-r', rev, '--template', template)

def decipher_hgapi_error (e):
	# Blah, hgapi, why must your error messages be so mangled?
	try:
		rawmsg = e.message.replace('\n', '').replace('" +','').replace('\t','')
		errmsg = re.compile (r'.*: tErr: (.*)Out:.*').match (rawmsg).group (1)
		return [True, errmsg]
	except:
		return [False, '']

def bbcodify (commit_diffstat):
	result = ''

	for line in commit_diffstat.split('\n'):
		rex = re.compile (r'^(.*)\|(.*) (\+*)(-*)(.*)$')
		match = rex.match (line)
		if match:
			line = '%s|%s [color=#5F7]%s[/color][color=#F53]%s[/color]%s\n' \
				% (match.group (1), match.group (2), match.group (3),
				match.group (4), match.group (5))

			# Tracker doesn't seem to like empty color tags
			line = line.replace ('[color=#5F7][/color]', '').replace ('[color=#F53][/color]', '')

		result += line

	return result

def poll():
	global repocheck_timeout
	if time.time() < repocheck_timeout:
		return

	for reponame in all_repo_names():
		poll_one_repo (reponame)

	hgns = Config.get_node ('hg')
	repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60

def poll_one_repo (repo_name):
	global repocheck_timeout
	hgns = Config.get_node ('hg')

	if not hgns.get_value ('track', default=True):
		return

	repo = hgapi.Repo (repo_name)
	commit_data = []
	delimeter = '^^^^^^^^^^'
	delimeter2 = '@@@@@@@@@@'
	print 'Checking %s for updates' % repo_name

	try:
		data = repo.hg_command ('incoming', '--quiet', '--template',
			delimeter.join (['{node|short}', '{desc}']) + delimeter2).split (delimeter2)
	except hgapi.hgapi.HgException as e:
		deciphered = decipher_hgapi_error (e)

		if deciphered[0] and len(deciphered[1]) > 0:
			Irc.broadcast ("error while using hg incoming on %s: %s" % (repo_name, deciphered[1]))

		return
	except Exception as e:
		Irc.broadcast ("%s" % `e`)
		return

	if not data:
		print ('No updates to %s' % repo_name)

	for line in data:
		if line:
			commit_data.append (line.split (delimeter))

	process_new_commits (repo_name, commit_data)

def process_new_commits (repo_name, commit_data):
	if len (commit_data) == 0:
		return

	repo_name = repo_name.lower()
	repo_url = get_repo_info (repo_name).get_value ('url')
	isExtraRepo = get_repo_info (repo_name).get_value ('extrarepo', False)
	zanrepo = hgapi.Repo (repo_name)
	print '%d new commits on %s' % (len (commit_data), repo_name)
	pull_args = []
	messages = [[], [], []]
	messageSizeClass = 2

	for commit in commit_data:
		pull_args.append ('-r');
		pull_args.append (commit[0]);

	print 'Pulling new commits...'
	try:
		zanrepo.hg_command ('pull', *pull_args)
	except Exception as e:
		Irc.broadcast ('Warning: unable to pull: %s' % `e`)
		return

	for commit in commit_data:
		commit_node = commit[0]
		commit_message = commit[1]
		print 'Processing new commit %s...' % commit_node

		try:
			existingrepos = g_CommitsDb.get_commit_repos (commit_node)
			alreadyAdded = len (existingrepos) > 0

			delim = '@@@@@@@@@@'
			data = get_commit_data (zanrepo, commit_node, delim.join (['{author}', '{bookmarks}', \
				'{date|hgdate}'])).split (delim)
			commit_author = data[0]
			commit_bookmarks = prettify_bookmarks (data[1])
			commit_time = int (data[2].split (' ')[0])
			commit_url = '%s/commits/%s' % (repo_url, commit_node)
			commit_email = ''

			# If the commit was already in the commits database, it is not a new one and we should
			# not react to it. Still add it to the db though so that the new repo name is added.
			g_CommitsDb.add_commit (repo=repo_name, changeset=commit_node, timestamp=commit_time)
			if alreadyAdded:
				print ('''I already know of %s - they're in %s - not announcing.''' %
					(commit_node, existingrepos))
				continue

			# Remove the email address from the author if possible
			rex = re.compile (r'^(.+) <([^>]+)>$.*')
			match = rex.match (commit_author)
			if match:
				commit_author = match.group (1)
				commit_email = match.group (2)

			commit_trackeruser = Config.find_developer_by_email (commit_email)
			committer = commit_trackeruser if commit_trackeruser != '' else commit_author

			commitDescriptor = """commit""" if int (math.ceil (random.random() * 100)) != 1 else """KERMIT"""

			commitMessage = """\003%d%s\003: new %s\0035 %s%s\003 by\0032 %s\003: %s""" % \
				(color_for_repo (repo_name), repo_name, commitDescriptor, commit_node, commit_bookmarks,
				committer, utility.shorten_link (commit_url))

			messages[0].append (commitMessage)
			messages[1].append (commitMessage)
			messages[2].append (commitMessage)
			messages[1].append ('   ' + commit_message.splitlines()[0])

			for line in commit_message.splitlines()[0:4]:
				messages[2].append ('    ' + line)

			if not isExtraRepo:
				rex = re.compile (r'^.*(fixes|resolves|addresses|should fix) ([0-9]+).*$')

				for line in commit_message.splitlines():
					match = rex.match (line)

					if match:
						announce_ticket_resolved (match.group (2), commit_node)
						break
		except Exception as e:
			Irc.broadcast ('Error while processing %s: %s' % (commit_node, e))
			continue

	fullMessageLength = len (''.join (messages[2]))

	if fullMessageLength > 3000:
		messageSizeClass = 0
	elif fullMessageLength > 768:
		messageSizeClass = 1

	print ("""Message length in total: %d, using size class %d (%d)""" %\
		(fullMessageLength, messageSizeClass, len (''.join (messages[messageSizeClass]))))

	# Post it all on IRC now
	for message in messages[messageSizeClass]:
		for irc_client in Irc.all_clients:
			for channel in irc_client.channels:
				if not channel.get_value ('btannounce', False):
					continue

				if isExtraRepo and not channel.get_value ('allpublishing', False):
					continue

				irc_client.privmsg (channel.get_value ('name'), message)

	# Hack for ZDoom upgrades
	if repo_name == 'zandronum-merge-experiments':
		check_zdoom_upgrade (repo_name, commit_node)

	g_CommitsDb.commit()

def make_progress_bar (p, barLength, colored=True):
	BoldChar, ColorChar = (Irc.BoldChar, Irc.ColorChar)
	return BoldChar + '[' + BoldChar \
	     + ColorChar + '2,2' + ('=' * int (round (p * barLength))) \
	     + ColorChar + '1,1' + ('-' * int (barLength - round (p * barLength))) \
	     + ColorChar + BoldChar + ']' + BoldChar

ZDoomMin = 2560
ZDoomMax = 3794

def check_zdoom_upgrade (repo_name, commit_node):
	zanrepo = hgapi.Repo (repo_name)
	try:
		global ZDoomRevNumber
		newnumber = get_zdrev_number (repo_name, commit_node)

		if newnumber > ZDoomRevNumber:
			ZDoomRevNumber = newnumber
			fraction = float (newnumber - ZDoomMin) / (ZDoomMax - ZDoomMin)
			# progressBar = make_progress_bar (fraction, 30, colored=True);

			message = """ZDoom upgrade: r%d: %d revisions left (%d%% complete)""" %\
				(newnumber, ZDoomMax - newnumber, round (fraction * 100))

			for irc_client in Irc.all_clients:
				for channel in irc_client.channels:
					if not channel.get_value ('allpublishing', False):
						continue
					irc_client.privmsg (channel.get_value ('name'), message)

			update_zadev_topic()
	except Exception as e:
		Irc.broadcast ('Error while dealing with ZDoom upgrade number: %s' % e)

def update_zadev_topic():
	fraction = float (ZDoomRevNumber - ZDoomMin) / (ZDoomMax - ZDoomMin)
	topicText = """ZDoom 2.6.1 progress: at r%d, %d revisions left (%d%% complete)""" %\
		(ZDoomRevNumber, ZDoomMax - ZDoomRevNumber, round (fraction * 100))

	try:
		Irc.ClientsByName['zandronum'].write ("""TOPIC #zadev :%s""" % topicText)
		Irc.ClientsByName['zandronum'].write ("""TOPIC #commits :%s""" % topicText)
	except Exception as e:
		Irc.broadcast ("""Error setting #zadev topic: %s""" % e)

def get_zdrev_number (repo_name, commit_node):
	zanrepo = hgapi.Repo (repo_name)
	subprocess.call (['hg', '--cwd', repo_name, 'revert', '-r', commit_node, 'src/version.h'])
	rx = re.compile (r'#define\s+ZD_SVN_REVISION_NUMBER\s+([0-9]+)')
	result = None

	with open (repo_name + '/src/version.h') as version_file:
		for line in version_file:
			match = rx.match (line)
			if match != None:
				result = int (match.group (1))
				break

	subprocess.call (['hg', '--cwd', repo_name, 'revert', '-r.', 'src/version.h'])

	if result != None:
		return result

	raise ValueError ("""Could not find ZD_SVN_REVISION_NUMBER in version.h""")

def force_poll():
	global repocheck_timeout
	repocheck_timeout = 0
	poll()