default/hgpoll.py

'''
	Copyright 2014-2015 Teemu Piippo
	All rights reserved.

	Redistribution and use in source and binary forms, with or without
	modification, are permitted provided that the following conditions
	are met:

	1. Redistributions of source code must retain the above copyright
	   notice, this list of conditions and the following disclaimer.
	2. Redistributions in binary form must reproduce the above copyright
	   notice, this list of conditions and the following disclaimer in the
	   documentation and/or other materials provided with the distribution.
	3. The name of the author may not be used to endorse or promote products
	   derived from this software without specific prior written permission.

	THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
	IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
	IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
	INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
	THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

from __future__ import print_function
import time
import re
import bt as Bt
import irc as Irc
import os
from configfile import Config
import utility
import random
from hgrepo import HgRepository
from hgdb import HgCommitsDatabase
import traceback
import sys
import datetime

Repositories = []
RepositoriesByName = {}

def prettify_bookmarks (bookmarks):
	if bookmarks:
		return "\0036 [\002%s\002]" % bookmarks
	else:
		return ''

def get_repo_by_name (name):
	global Repositories, RepositoriesByName
	name = name.lower()

	if name not in RepositoriesByName:
		repo = HgRepository (name)
		Repositories.append (repo)
		RepositoriesByName[name] = repo
	else:
		repo = RepositoriesByName[name]

	return repo

def check_repo_exists (name):
	' Ensures that the repository exists '
	repo = get_repo_by_name (name)
	os.makedirs (repo.name, exist_ok = True)

	if not repo.is_valid():
		# If the repo does not exist, clone it.
		try:
			repo.clone()

			# We need to un-alias a few things in case they're aliased
			comms=['log', 'incoming', 'pull', 'commit', 'push', 'outgoing', 'strip', 'transplant']
			try:
				with open (os.path.join (repo.name, '.hg', 'hgrc'), 'a') as fp:
					fp.write ('\n[alias]\n' + ''.join('%s=%s\n' % (x, x) for x in comms))
			except Exception as e:
				print ('Warning: unable to alter hgrc of %s: %s' % repo.name, e)
			print ('Cloning done.')
		except Exception as e:
			raise HgProcessError ('Unable to clone %s from %s: %s' % (repo.name, repo.clone_url, e))

	if not repo.is_valid():
		raise HgProcessError ('''%s is not a valid repository after cloning '''
			'''(.hg is missing)''' % repo.name)

class HgProcessError (Exception):
	def __init__ (self, value):
		self.message = value
	def __str__ (self):
		return self.message

def contains_published_repositories (repos):
	for repo in repos:
		if repo.published:
			return True

	return False

def announce_ticket_resolved (ticket_id, cset, db):
	ticket_id = int (ticket_id)
	repos = db.get_commit_repos (cset)

	for repo in repos:
		if repo.published:
			break
	else:
		raise HgProcessError ('Changeset %s is only committed to non-published repositories: %s' %
			(cset, ', '.join (repos)))

	# Acquire additional data
	commit = repo.get_commit_data (rev=cset,
		author='author|person',
		date='date|hgdate',
		email='author|email',
		message='desc')

	commit['date'] = datetime.datetime.utcfromtimestamp(int(commit['date'][0]) - int(commit['date'][1]))
	commit['date'] = commit['date'].strftime('%A %d %B %Y %H:%M:%S')

	diffstat = repo.hg ('diff', '--change', cset, '--stat')

	try:
		ticket_data = Bt.get_issue (ticket_id)
	except Exception as e:
		raise HgProcessError ("error while processing %s: %s" % (cset, e))

	if len(diffstat) > 0:
		diffstat = 'Changes in files:\n[code]\n' + diffstat + '\n[/code]'
	else:
		diffstat = 'No changes in files.'

	# Compare the email addresses against known developer usernames
	username = Config.find_developer_by_email (commit['email'])

	if username:
		commit['author'] += ' [%s]' % username

	commit['message'] = commit['message'].replace ("\n", " ")

	message = 'Issue addressed by commit %s: [b][url=%s/commits/%s]%s[/url][/b]' \
		% (cset, repo.url, cset, commit['message'])
	message += "\nCommitted by %s on %s\n\n%s" \
			% (commit['author'], commit['date'], diffstat)

	need_update = False

	# If not already set, set handler
	if username and not 'handler' in ticket_data:
		ticket_data['handler'] = {'name': username}
		need_update = True

	# Find out the status level of the ticket
	needs_testing_level = 70

	if ticket_data['status']['id'] < needs_testing_level:
		ticket_data.status['id'] = needs_testing_level
		need_update = True

	# Set target version if not set
	if repo.version and not 'target_version' in ticket_data:
		ticket_data['target_version'] = repo.version
		need_update = True

	# Announce on IRC
	for irc_client in Irc.all_clients:
		for channel in irc_client.channels:
			if channel.get_value ('btannounce', default=True):
				irc_client.privmsg (channel.get_value ('name'),
					"\003%d%s\003: commit\0035 %s\003 addresses issue\002\0032 %d\002\003: %s" % \
					(repo.color, repo.name, cset, ticket_id, Bt.get_ticket_url(ticket_id)))

	if need_update:
		# We need to remove the note data, otherwise the ticket notes
		# will get unnecessary updates. WTF, MantisBT?
		ticket_data.notes = []
		Bt.update_issue (ticket_id, ticket_data)

	Bt.post_note (ticket_id, message)

def init():
	global repocheck_timeout

	for name in Config.get_node ('hg').get_value ('repos', {}).keys():
		check_repo_exists (name)

	# Let the database check if commits.db needs to be built
	HgCommitsDatabase()

	repocheck_timeout = time.time() + 15

def poll():
	global repocheck_timeout

	try:
		if time.time() < repocheck_timeout:
			return

		hgns = Config.get_node ('hg')
		repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60
		maxcommits = 15
		numcommits = 0

		for repo in Repositories:
			numcommits += poll_one_repo (repo, maxcommits=maxcommits - numcommits)

			if numcommits >= maxcommits:
				# There may be more coming so recheck sooner				
				print ('Processed %d commits, checking for new commits in 1 minute...' % numcommits)
				repocheck_timeout = time.time() + 60
				return
	except Exception as e:
		print (traceback.format_exception (*sys.exc_info()))
		Irc.broadcast (str (e))

def poll_one_repo (repo, maxcommits):
	if not Config.get_node ('hg').get_value ('track', default=True):
		return

	print ('Checking %s for updates' % repo.name)
	commits = repo.incoming (maxcommits=maxcommits, node='node|short', message='desc')
	process_new_commits (repo, commits)
	return len(commits)

def process_new_commits (repo, commits):
	if not commits:
		return

	print ('%d new commits on %s' % (len (commits), repo.name))
	pull_args = ['pull']
	messages = [[], [], []]

	for commit in commits:
		pull_args.append ('-r');
		pull_args.append (commit['node']);

	print ('Pulling new commits...')
	repo.hg (*pull_args)
	LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL = range (0, 3)
	db = HgCommitsDatabase()

	for commit in commits:
		print ('Processing new commit %s...' % commit['node'])

		existingrepos = db.get_commit_repos (commit['node'])
		alreadyAdded = len (existingrepos) > 0

		commit = dict (commit, **repo.get_commit_data (rev=commit['node'],
			fullnode='node',
			author='author|person',
			bookmarks='bookmarks',
			date='date|hgdate',
			email='author|email'))
		commit['bookmarks'] = prettify_bookmarks (commit['bookmarks'])
		commit['time'] = int (commit['date'].split (' ')[0])
		commit['url'] = '%s/commits/%s' % (repo.url, commit['node'])
		isMergeFromSandbox = False

		# If the commit was already in the commits database, it is not a new one and we should
		# not react to it (unless a merge from sandbox). Still add it to the db though so that
		# the new repo name is added.
		db.add_commit (repo=repo, changeset=commit['fullnode'],
			timestamp=commit['time'])

		if alreadyAdded:
			if not contains_published_repositories (existingrepos) and repo.published:
				isMergeFromSandbox = True
				print ('''%s appears to be a merge from sandbox (exists in %s)''' %
					(commit['node'], existingrepos))
			else:
				print ('''I already know of %s - they're in %s - not announcing.''' %
					(commit['node'], existingrepos))
				continue

		username = Config.find_developer_by_email (commit['email'])
		committer = username if username else commit['author']
		descriptor = """commit""" if random.randrange (100) != 0 else """KERMIT"""

		if not isMergeFromSandbox:
			commitMessage = """\003%d%s\003: new %s\0035 %s%s\003 by\0032 %s\003: %s""" % \
				(repo.color, repo.name, descriptor, commit['node'], commit['bookmarks'],
				committer, utility.shorten_link (commit['url']))

			multiline = ('\n' in commit['message']) or len(commit['message']) >= 256

			if not multiline:
				commitMessage += ": %s" % commit['message']

			for length in [LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL]:
				messages[length].append (commitMessage)

			if multiline:
				messages[LENGTH_SHORT].append ('   ' + commit['message'].splitlines()[0])

				for line in commit['message'].splitlines()[:4]:
					messages[LENGTH_FULL].append ('    ' + line)
		else:
			commitMessage = """\003%d%s\003: %s\0035 %s\003 by\0032 %s\003 was pulled: %s""" % \
				(repo.color, repo.name, descriptor, commit['node'], committer,
				utility.shorten_link (commit['url']))

			for length in [LENGTH_MINIMUM, LENGTH_SHORT, LENGTH_FULL]:
				messages[length].append (commitMessage)

		if repo.published:
			rex = re.compile (r'^.*(fixes|resolves|addresses|should fix) ([0-9]+).*$')

			for line in commit['message'].splitlines():
				match = rex.match (line)

				if match:
					announce_ticket_resolved (match.group (2), commit['node'], db)
					break

	fullMessageLength = len (''.join (messages[2]))

	if fullMessageLength > 3000:
		messageSizeClass = LENGTH_MINIMUM
	elif fullMessageLength > 768:
		messageSizeClass = LENGTH_SHORT
	else:
		messageSizeClass = LENGTH_FULL

	# Post it all on IRC now
	for irc_client in Irc.all_clients:
		for channel in irc_client.channels:
			if not channel.get_value ('btannounce', False):
				continue

			if not repo.published and not channel.get_value ('allpublishing', False):
				continue

			for message in messages[messageSizeClass]:
				irc_client.privmsg (channel.get_value ('name'), message)

	db.commit()

def force_poll():
	global repocheck_timeout
	repocheck_timeout = 0
	poll()