1 import hgapi |
1 import hgapi |
2 import time |
2 import time |
3 import re |
3 import re |
4 import bt as Bt |
4 import bt as Bt |
5 import irc as Irc |
5 import irc as Irc |
6 import os |
6 import os, sqlite3, subprocess |
7 from datetime import datetime |
7 from datetime import datetime |
8 from configfile import Config |
8 from configfile import Config |
9 import utility |
9 import utility |
10 g_needCommitsTxtRebuild = True |
10 g_needCommitsTxtRebuild = True |
11 |
11 |
|
12 class CommitsDb (object): |
|
13 def __init__(self): |
|
14 needNew = not os.path.isfile ('commits.db') |
|
15 self.db = sqlite3.connect ('commits.db') |
|
16 |
|
17 if needNew: |
|
18 self.createNew() |
|
19 |
|
20 def create_new (self): |
|
21 self.db.executescript (''' |
|
22 DROP TABLE IF EXISTS COMMITS; |
|
23 DROP TABLE IF EXISTS REPOS; |
|
24 DROP TABLE IF EXISTS REPOCOMMITS; |
|
25 CREATE TABLE IF NOT EXISTS COMMITS |
|
26 ( |
|
27 Node text NOT NULL, |
|
28 Dateversion text NOT NULL, |
|
29 PRIMARY KEY (Node) |
|
30 ); |
|
31 CREATE TABLE IF NOT EXISTS REPOS |
|
32 ( |
|
33 Name text NOT NULL, |
|
34 PRIMARY KEY (Name) |
|
35 ); |
|
36 CREATE TABLE IF NOT EXISTS REPOCOMMITS |
|
37 ( |
|
38 Reponame text, |
|
39 Node text, |
|
40 FOREIGN KEY (Reponame) REFERENCES REPOS(Name), |
|
41 FOREIGN KEY (Node) REFERENCES COMMITS(Node) |
|
42 ); |
|
43 ''') |
|
44 |
|
45 print 'Building commits.db...' |
|
46 for repo in all_repo_names(): |
|
47 print 'Adding commits from %s...' % repo |
|
48 |
|
49 data = subprocess.check_output (['hg', '--cwd', repo, 'log', '--template', |
|
50 '{node} {date|hgdate}\n']).splitlines() |
|
51 |
|
52 for line in data: |
|
53 changeset, timestamp, tz = line.split(' ') |
|
54 self.add_commit (repo, changeset, int (timestamp)) |
|
55 |
|
56 self.commit() |
|
57 |
|
58 def add_commit (self, repo, changeset, timestamp): |
|
59 dateversion = datetime.utcfromtimestamp (timestamp).strftime ('%y%m%d-%H%M') |
|
60 self.db.execute (''' |
|
61 INSERT OR IGNORE INTO REPOS |
|
62 VALUES (?) |
|
63 ''', (repo,)) |
|
64 |
|
65 self.db.execute (''' |
|
66 INSERT OR IGNORE INTO COMMITS |
|
67 VALUES (?, ?) |
|
68 ''', (changeset, dateversion)) |
|
69 |
|
70 self.db.execute (''' |
|
71 INSERT INTO REPOCOMMITS |
|
72 VALUES (?, ?) |
|
73 ''', (repo, changeset)) |
|
74 |
|
75 def get_commit_repos (self, node): |
|
76 cursor = self.db.execute (''' |
|
77 SELECT Reponame |
|
78 FROM REPOCOMMITS |
|
79 WHERE Node LIKE ? |
|
80 ''', (node + '%',)) |
|
81 |
|
82 results = cursor.fetchall() |
|
83 return list (set (zip (*results)[0])) if results else [] |
|
84 |
|
85 def find_commit_by_dateversion (self, dateversion): |
|
86 cursor = self.db.execute (''' |
|
87 SELECT Node |
|
88 FROM COMMITS |
|
89 WHERE Dateversion = ? |
|
90 ''', (dateversion,)) |
|
91 result = cursor.fetchone() |
|
92 return result[0] if result else None |
|
93 |
|
94 def commit(self): |
|
95 self.db.commit() |
|
96 |
|
97 g_CommitsDb = CommitsDb() |
|
98 |
12 def color_for_repo (repo_name): |
99 def color_for_repo (repo_name): |
13 repo_name = repo_name.lower() |
100 repo_name = repo_name.lower() |
14 |
101 repoconfig = Config.get_node ('hg').get_node ('repos') |
15 if repo_name == 'zandronum': |
102 |
16 color = 4 |
103 if repoconfig.has_node (repo_name): |
17 elif repo_name == 'zandronum-stable': |
104 return repoconfig.get_node(repo_name).get_value ('colorcode', 0) |
18 color = 2 |
105 |
19 elif repo_name == 'zandronum-sandbox': |
106 return 0 |
20 color = 7 |
|
21 elif repo_name == 'zandronum-sandbox-stable': |
|
22 color = 3 |
|
23 elif repo_name == 'zfc9000': |
|
24 color = 6 |
|
25 elif repo_name == 'doomseeker': |
|
26 color = 5 |
|
27 else: |
|
28 color = 0 |
|
29 return color |
|
30 |
107 |
31 def prettify_bookmarks (bookmarks): |
108 def prettify_bookmarks (bookmarks): |
32 return "\0036 [\002%s\002]" % bookmarks |
109 return "\0036 [\002%s\002]" % bookmarks |
33 |
110 |
34 def make_commits_txt(): |
111 def get_repo_info (reponame): |
35 global g_needCommitsTxtRebuild |
112 reponame = reponame.lower() |
36 |
113 repoconfig = Config.get_node ('hg').get_node ('repos') |
37 if g_needCommitsTxtRebuild == False: |
114 |
38 return |
115 if repoconfig.has_node (reponame): |
39 |
116 return repoconfig.get_node (reponame) |
40 print 'Building commits.txt...' |
117 |
41 # Update zandronum-everything |
118 return None |
42 repo = hgapi.Repo ('zandronum-everything') |
|
43 repo.hg_command ('pull', '../zandronum-sandbox') |
|
44 repo.hg_command ('pull', '../zandronum-sandbox-stable') |
|
45 data = repo.hg_command ('log', '--template', '{node} {date|hgdate}\n') |
|
46 f = open ('commits.txt', 'w') |
|
47 |
|
48 for line in data.split ('\n'): |
|
49 if line == '': |
|
50 continue |
|
51 |
|
52 words = line.split (' ') |
|
53 timestamp = int (words[1]) |
|
54 f.write ('%s %s\n' % (words[0], datetime.utcfromtimestamp (timestamp).strftime ('%y%m%d-%H%M'))) |
|
55 |
|
56 f.close() |
|
57 g_needCommitsTxtRebuild = False |
|
58 |
119 |
59 ' Check if a repository exists ' |
120 ' Check if a repository exists ' |
60 def check_repo_exists (repo_name, repo_owner): |
121 def check_repo_exists (repo_name): |
61 print 'Checking that %s exists...' % repo_name |
122 print 'Checking that %s exists...' % repo_name |
62 repo_url = 'https://bitbucket.org/%s/%s' % (repo_owner, repo_name) |
|
63 zanrepo = hgapi.Repo (repo_name) |
123 zanrepo = hgapi.Repo (repo_name) |
|
124 |
|
125 if not os.path.exists (repo_name): |
|
126 os.makedirs (repo_name) |
64 |
127 |
65 try: |
128 try: |
66 zanrepo.hg_command ('id', '.') |
129 zanrepo.hg_command ('id', '.') |
67 except hgapi.hgapi.HgException: |
130 except hgapi.hgapi.HgException: |
68 # If the repo does not exist, clone it. zandronum-everything can be spawned |
131 # If the repo does not exist, clone it. |
69 # off other repos though |
132 repo_url = get_repo_info (repo_name).get_value ('url') |
70 if repo_name == 'zandronum-everything': |
|
71 if not os.path.exists (repo_name): |
|
72 os.makedirs (repo_name) |
|
73 |
|
74 global g_needCommitsTxtRebuild |
|
75 g_needCommitsTxtRebuild = True |
|
76 print 'Init %s' % repo_name |
|
77 zanrepo.hg_command ('init') |
|
78 print 'Cloning zandronum-sandbox into %s' % repo_name |
|
79 zanrepo.hg_command ('pull', '../zandronum-sandbox') |
|
80 print 'Cloning zandronum-sandbox-stable into %s' % repo_name |
|
81 zanrepo.hg_command ('pull', '../zandronum-sandbox-stable') |
|
82 print 'Done' |
|
83 make_commits_txt() |
|
84 return |
|
85 |
|
86 try: |
133 try: |
87 print 'Cloning %s...' % repo_name |
134 print ('Cloning %s...' % repo_name) |
88 zanrepo.hg_clone (repo_url, repo_name) |
135 zanrepo.hg_clone (repo_url, repo_name) |
89 print 'Cloning done.' |
136 print ('Cloning done.') |
90 except Exception as e: |
137 except Exception as e: |
91 print 'Unable to clone %s from %s: %s' % (repo_name, repo_url, str (`e`)) |
138 print ('Unable to clone %s from %s: %s' % (repo_name, repo_url, e)) |
92 quit (1) |
139 quit (1) |
93 |
140 |
94 |
141 |
95 class HgProcessError (Exception): |
142 class HgProcessError (Exception): |
96 def __init__ (self, value): |
143 def __init__ (self, value): |
97 self.message = value |
144 self.message = value |
98 def __str__ (self): |
145 def __str__ (self): |
99 return self.message |
146 return self.message |
100 |
147 |
101 def announce_ticket_resolved (ticket_id, commit_node): |
148 def announce_ticket_resolved (ticket_id, cset): |
102 ticket_id = int (ticket_id) |
149 ticket_id = int (ticket_id) |
103 |
150 reponames = g_CommitsDb.get_commit_repos (cset) |
104 try: |
151 |
105 repo_name = 'zandronum-stable' |
152 if not reponames: |
106 zanrepo = hgapi.Repo (repo_name) |
153 raise HgProcessError ('Changeset %s does not appear to exist!' % cset) |
107 zanrepo.hg_command ('log', '-r', commit_node) |
154 |
108 except hgapi.hgapi.HgException: |
155 for reponame in reponames: |
109 try: |
156 repoinfo = get_repo_info (reponame) |
110 repo_name = 'zandronum' |
157 if not repoinfo: |
111 zanrepo = hgapi.Repo (repo_name) |
158 raise HgProcessError ('Unknown repo %s' % reponame) |
112 zanrepo.hg_command ('log', '-r', commit_node) |
159 |
113 except hgapi.hgapi.HgException: |
160 if not repoinfo.get_value ('private', default=False): |
114 raise HgProcessError ('unknown commit %s' % commit_node) |
161 break |
115 |
162 else: |
116 repo_url = 'https://bitbucket.org/Torr_Samaho/%s' % repo_name |
163 raise HgProcessError ('Changeset %s is only committed to non-published repositories %s' % |
|
164 (cset, reponames)) |
|
165 |
|
166 repo = hgapi.Repo (reponame) |
|
167 repo_url = repoinfo.get_value ('url', default=None) |
|
168 |
|
169 if not repo_url: |
|
170 raise HgProcessError ('Repo %s has no url!' % reponame) |
117 |
171 |
118 # Acquire additional data |
172 # Acquire additional data |
119 moredata = get_commit_data (zanrepo, commit_node, |
173 moredata = get_commit_data (repo, cset, |
120 '{author|nonempty}\n{date(date, \'%A %d %B %Y %T\')}').split('\n') |
174 r"{author|nonempty}\n{date(date, '%A %d %B %Y %H:%M:%S')}").split('\n') |
121 |
175 |
122 if len (moredata) != 2: |
176 if len (moredata) != 2: |
123 raise HgProcessError ('malformed hg data while processing %s' % commit_node) |
177 raise HgProcessError ('malformed hg data while processing %s' % cset) |
124 |
178 |
125 commit_author = moredata[0] |
179 commit_author = moredata[0] |
126 commit_date = moredata[1] |
180 commit_date = moredata[1] |
127 commit_email = "" |
181 commit_email = "" |
128 commit_message = zanrepo.hg_command ('log', '-r', commit_node, '--template', '{desc}') |
182 commit_message = repo.hg_command ('log', '-r', cset, '--template', '{desc}') |
129 |
183 |
130 try: |
184 try: |
131 ticket_data = Bt.get_issue (ticket_id) |
185 ticket_data = Bt.get_issue (ticket_id) |
132 except Exception as e: |
186 except Exception as e: |
133 raise HgProcessError ("error while processing %s: %s" % (commit_node, e)) |
187 raise HgProcessError ("error while processing %s: %s" % (cset, e)) |
134 |
188 |
135 # Remove the email address from the author if possible |
189 # Remove the email address from the author if possible |
136 rex = re.compile (r'^(.+) <([^>]+)>$.*') |
190 rex = re.compile (r'^(.+) <([^>]+)>$.*') |
137 match = rex.match (commit_author) |
191 match = rex.match (commit_author) |
138 if match: |
192 if match: |
139 commit_author = match.group (1) |
193 commit_author = match.group (1) |
140 commit_email = match.group (2) |
194 commit_email = match.group (2) |
141 |
195 |
142 commit_diffstat = zanrepo.hg_command ('diff', '--change', commit_node, '--stat') |
196 commit_diffstat = repo.hg_command ('diff', '--change', cset, '--stat') |
143 |
197 |
144 if len(commit_diffstat) > 0: |
198 if len(commit_diffstat) > 0: |
145 # commit_diffstat = 'Changes in files:\n[code]\n' + commit_diffstat + '\n[/code]' |
199 # commit_diffstat = 'Changes in files:\n[code]\n' + commit_diffstat + '\n[/code]' |
146 commit_diffstat = 'Changes in files:\n' + bbcodify(commit_diffstat) |
200 commit_diffstat = 'Changes in files:\n' + bbcodify(commit_diffstat) |
147 else: |
201 else: |
203 ticket_data.notes = [] |
248 ticket_data.notes = [] |
204 Bt.update_issue (ticket_id, ticket_data) |
249 Bt.update_issue (ticket_id, ticket_data) |
205 |
250 |
206 Bt.post_note (ticket_id, message) |
251 Bt.post_note (ticket_id, message) |
207 |
252 |
|
253 def all_repo_names(): |
|
254 return Config.get_node ('hg').get_value ('repos', {}).keys() |
|
255 |
208 def init(): |
256 def init(): |
209 check_repo_exists ('zandronum', 'Torr_Samaho') |
257 for repo in all_repo_names(): |
210 check_repo_exists ('zandronum-stable', 'Torr_Samaho') |
258 check_repo_exists (repo) |
211 check_repo_exists ('zandronum-sandbox', 'crimsondusk') |
|
212 check_repo_exists ('zandronum-sandbox-stable', 'crimsondusk') |
|
213 check_repo_exists ('zandronum-everything', '') |
|
214 check_repo_exists ('zfc9000', 'crimsondusk') |
|
215 check_repo_exists ('doomseeker', 'Blzut3') |
|
216 global repocheck_timeout |
259 global repocheck_timeout |
217 repocheck_timeout = (time.time()) + 15 |
260 repocheck_timeout = time.time() + 15 |
218 |
261 |
219 def get_commit_data (zanrepo, rev, template): |
262 def get_commit_data (repo, rev, template): |
220 return zanrepo.hg_command ('log', '-l', '1', '-r', rev, '--template', template) |
263 return repo.hg_command ('log', '-l', '1', '-r', rev, '--template', template) |
221 |
264 |
222 def decipher_hgapi_error (e): |
265 def decipher_hgapi_error (e): |
223 # Blah, hgapi, why must your error messages be so mangled? |
266 # Blah, hgapi, why must your error messages be so mangled? |
224 try: |
267 try: |
225 rawmsg = e.message.replace('\n', '').replace('" +','').replace('\t','') |
268 rawmsg = e.message.replace('\n', '').replace('" +','').replace('\t','') |
249 def poll(): |
292 def poll(): |
250 global repocheck_timeout |
293 global repocheck_timeout |
251 if time.time() < repocheck_timeout: |
294 if time.time() < repocheck_timeout: |
252 return |
295 return |
253 |
296 |
254 for n in ['zandronum-stable', 'zandronum', 'zandronum-sandbox', 'zandronum-sandbox-stable', 'zfc9000', 'doomseeker']: |
297 for reponame in all_repo_names(): |
255 poll_one_repo (n) |
298 poll_one_repo (reponame) |
256 |
299 |
257 hgns = Config.get_node ('hg') |
300 hgns = Config.get_node ('hg') |
258 repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60 |
301 repocheck_timeout = time.time() + hgns.get_value ('checkinterval', default=15) * 60 |
259 |
|
260 def get_repo_owner (repo_name): |
|
261 usesandbox = repo_name == 'zandronum-sandbox' or repo_name == 'zandronum-sandbox-stable' |
|
262 repo_name = repo_name.lower() |
|
263 if repo_name[:9] == 'zandronum': |
|
264 repo_owner = 'Torr_Samaho' if (not usesandbox) else 'crimsondusk' |
|
265 elif repo_name == 'zfc9000': |
|
266 repo_owner = 'crimsondusk' |
|
267 elif repo_name == 'doomseeker': |
|
268 repo_owner = 'Blzut3' |
|
269 return repo_owner |
|
270 |
302 |
271 def poll_one_repo (repo_name): |
303 def poll_one_repo (repo_name): |
272 global repocheck_timeout |
304 global repocheck_timeout |
273 hgns = Config.get_node ('hg') |
305 hgns = Config.get_node ('hg') |
274 |
306 |
275 if not hgns.get_value ('track', default=True): |
307 if not hgns.get_value ('track', default=True): |
276 return |
308 return |
277 |
309 |
278 usesandbox = repo_name == 'zandronum-sandbox' or repo_name == 'zandronum-sandbox-stable' |
310 repo = hgapi.Repo (repo_name) |
279 repo_owner = get_repo_owner (repo_name) |
|
280 zanrepo = hgapi.Repo (repo_name) |
|
281 commit_data = [] |
311 commit_data = [] |
282 delimeter = '@@@@@@@@@@' |
312 delimeter = '@@@@@@@@@@' |
283 # print 'Checking %s for updates' % repo_name |
313 print 'Checking %s for updates' % repo_name |
284 |
314 |
285 try: |
315 try: |
286 data = zanrepo.hg_command ('incoming', '--quiet', '--template', |
316 data = repo.hg_command ('incoming', '--quiet', '--template', |
287 '{node|short} {desc}' + delimeter) |
317 '{node|short} {desc}' + delimeter).split (delimeter) |
288 except hgapi.hgapi.HgException as e: |
318 except hgapi.hgapi.HgException as e: |
289 deciphered = decipher_hgapi_error (e) |
319 deciphered = decipher_hgapi_error (e) |
290 |
320 |
291 if deciphered[0] and len(deciphered[1]) > 0: |
321 if deciphered[0] and len(deciphered[1]) > 0: |
292 Irc.broadcast ("error while using hg import on %s: %s" % (repo_name, deciphered[1])) |
322 Irc.broadcast ("error while using hg import on %s: %s" % (repo_name, deciphered[1])) |
343 pull_args.append (commit[0]); |
374 pull_args.append (commit[0]); |
344 |
375 |
345 print 'Pulling new commits...' |
376 print 'Pulling new commits...' |
346 try: |
377 try: |
347 zanrepo.hg_command ('pull', *pull_args) |
378 zanrepo.hg_command ('pull', *pull_args) |
348 |
|
349 if repo_name[0:9] == 'zandronum': |
|
350 # Also pull these commits to the zandronum main repository |
|
351 if usestable: |
|
352 devrepo = hgapi.Repo ('zandronum') |
|
353 devrepo.hg_command ('pull', '../zandronum-stable', *pull_args) |
|
354 # Pull everything into sandboxes too |
|
355 if not usesandbox: |
|
356 devrepo = hgapi.Repo ('zandronum-sandbox') |
|
357 devrepo.hg_command ('pull', '../%s' % repo_name, *pull_args) |
|
358 |
|
359 devrepo = hgapi.Repo ('zandronum-sandbox-stable') |
|
360 devrepo.hg_command ('pull', '../%s' % repo_name, *pull_args) |
|
361 |
|
362 devrepo = hgapi.Repo ('zandronum-everything') |
|
363 devrepo.hg_command ('pull', '../%s' % repo_name, *pull_args) |
|
364 |
|
365 global g_needCommitsTxtRebuild |
|
366 g_needCommitsTxtRebuild = True |
|
367 |
|
368 except Exception as e: |
379 except Exception as e: |
369 Irc.broadcast ('Warning: unable to pull: %s' % `e`) |
380 Irc.broadcast ('Warning: unable to pull: %s' % `e`) |
370 return |
381 return |
371 |
382 |
372 for commit in commit_data: |
383 for commit in commit_data: |
373 commit_node = commit[0] |
384 commit_node = commit[0] |
374 commit_message = commit[1] |
385 commit_message = commit[1] |
375 print 'Processing new commit %s...' % commit_node |
386 print 'Processing new commit %s...' % commit_node |
376 |
387 |
377 try: |
388 try: |
378 data = get_commit_data (zanrepo, commit_node, '{author}@@@@@@@@@@{bookmarks}').split ("@@@@@@@@@@") |
389 alreadyAdded = len (g_CommitsDb.get_commit_repos (commit_node)) > 0 |
|
390 |
|
391 delim = '@@@@@@@@@@' |
|
392 data = get_commit_data (zanrepo, commit_node, delim.join (['{author}', '{bookmarks}', \ |
|
393 '{date|hgdate}'])).split (delim) |
|
394 print 'data is: %s' % data |
379 commit_author = data[0] |
395 commit_author = data[0] |
380 commit_bookmarks = prettify_bookmarks (data[1]) |
396 commit_bookmarks = prettify_bookmarks (data[1]) |
|
397 commit_time = int (data[2].split (' ')[0]) |
381 commit_url = '%s/commits/%s' % (repo_url, commit_node) |
398 commit_url = '%s/commits/%s' % (repo_url, commit_node) |
382 commit_email = '' |
399 commit_email = '' |
|
400 |
|
401 # If the commit was already in the commits database, it is not a new one and we should |
|
402 # not react to it. Still add it to the db though so that the new repo name is added. |
|
403 g_CommitsDb.add_commit (repo=repo_name, changeset=commit_node, timestamp=commit_time) |
|
404 if alreadyAdded: |
|
405 continue |
383 |
406 |
384 # Remove the email address from the author if possible |
407 # Remove the email address from the author if possible |
385 rex = re.compile (r'^(.+) <([^>]+)>$.*') |
408 rex = re.compile (r'^(.+) <([^>]+)>$.*') |
386 match = rex.match (commit_author) |
409 match = rex.match (commit_author) |
387 if match: |
410 if match: |