testsuite.py

Fri, 24 May 2019 15:32:10 +0300

author
Teemu Piippo <teemu@hecknology.net>
date
Fri, 24 May 2019 15:32:10 +0300
changeset 37
e46fa477007b
parent 32
75f44d3063da
child 47
4da025d0b283
permissions
-rw-r--r--

Fix operation under Python 3.4

16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
1 from warnings import warn
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
2
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
3 def report_element(bad_object, type, error_name, args):
12
eb74680a5e43 commit work done on test suites...
Santeri Piippo
parents:
diff changeset
4 return {
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
5 'type': type,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
6 'object': bad_object,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
7 'name': error_name,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
8 'args': args,
13
12d4ddc4bfd8 Got the skew test working
Santeri Piippo
parents: 12
diff changeset
9 }
12d4ddc4bfd8 Got the skew test working
Santeri Piippo
parents: 12
diff changeset
10
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
11 def warning(bad_object, error_name, **args):
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
12 return report_element(bad_object, 'warning', error_name, args)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
13
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
14 def error(bad_object, error_name, **args):
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
15 return report_element(bad_object, 'error', error_name, args)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
16
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
17 def notice(bad_object, error_name, **args):
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
18 return report_element(bad_object, 'notice', error_name, args)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
19
37
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
20 def name_of_package(package):
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
21 if isinstance(package, tuple):
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
22 return package[1]
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
23 else:
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
24 return package.name
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
25
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
26 def test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
27 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
28 Finds all test modules and yields their names.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
29 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
30 from pkgutil import walk_packages
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
31 import tests
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
32 yield from sorted(
37
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
33 'tests.' + name_of_package(result)
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
34 for result in walk_packages(tests.__path__)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
35 )
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
36
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
37 def do_manifest_integrity_checks(test_suite, module):
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
38 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
39 Runs integrity checks on a given module's manifest.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
40 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
41 def check_for_extra_keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
42 extra_keys = module.manifest.keys() - test_suite.keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
43 if extra_keys:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
44 warn(str.format(
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
45 '{}: extra keys in manifest: {}',
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
46 module.__name__,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
47 ', '.join(map(str, extra_keys))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
48 ))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
49 def check_for_manifest_duplicates():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
50 for key in test_suite.keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
51 duplicates = module.manifest[key].keys() & test_suite[key].keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
52 if duplicates:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
53 warn(str.format(
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
54 '{}: redefined {} in manifests: {}',
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
55 module.__name__,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
56 key,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
57 duplicates,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
58 ))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
59 check_for_extra_keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
60 check_for_manifest_duplicates()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
61
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
62 def load_tests():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
63 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
64 Imports test modules and combines their manifests into a test suite.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
65 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
66 test_suite = {'tests': {}, 'messages': {}}
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
67 for module_name in test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
68 from importlib import import_module
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
69 module = import_module(module_name)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
70 if hasattr(module, 'manifest'):
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
71 do_manifest_integrity_checks(test_suite, module)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
72 # Merge the data from the manifest
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
73 for key in module.manifest.keys() & test_suite.keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
74 test_suite[key].update(module.manifest[key])
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
75 else:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
76 warn(str.format('Module {} does not have a manifest', module_name))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
77 return test_suite
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
78
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
79 def problem_key(problem):
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
80 problem_hierarchy = ['error', 'warning', 'notice']
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
81 return (problem_hierarchy.index(problem['type']), problem['line-number'])
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
82
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
83 def check_model(model, test_suite = None):
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
84 if not test_suite:
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
85 test_suite = load_tests()
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
86 problems = []
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
87 line_numbers = {
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
88 element: (i, i + 1 + model.body_offset)
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
89 for i, element in enumerate(model.body)
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
90 }
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
91 for test_name, test_function in test_suite['tests'].items():
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
92 for problem in test_function(model):
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
93 problem['body-index'], problem['line-number'] \
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
94 = line_numbers[problem['object']]
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
95 del problem['object']
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
96 problems.append(problem)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
97 return {
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
98 'passed': not any(
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
99 problem['type'] == 'error'
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
100 for problem in problems
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
101 ),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
102 'problems': sorted(problems, key = problem_key),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
103 }
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
104
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
105 def problem_text(problem, test_suite):
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
106 message = test_suite['messages'][problem['name']]
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
107 if callable(message):
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
108 message = message(**problem['args'])
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
109 return message
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
110
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
111 def format_report_html(report, model, test_suite):
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
112 messages = []
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
113 for problem in report['problems']:
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
114 ldraw_code = model.body[problem['body-index']].textual_representation()
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
115 message = str.format(
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
116 '<li class="{problem_type}">{model_name}:{line_number}:'
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
117 '{problem_type}: {message}<br />{ldraw_code}</li>',
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
118 model_name = model.name,
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
119 line_number = problem['line-number'],
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
120 problem_type = problem['type'],
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
121 message = problem_text(problem, test_suite),
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
122 ldraw_code = ldraw_code,
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
123 )
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
124 messages.append(message)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
125 return '\n'.join(messages)
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
126
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
127 def format_report(report, model, test_suite):
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
128 import colorama
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
129 colorama.init()
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
130 messages = []
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
131 for problem in report['problems']:
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
132 if problem['type'] == 'error':
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
133 text_colour = colorama.Fore.LIGHTRED_EX
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
134 elif problem['type'] == 'warning':
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
135 text_colour = colorama.Fore.LIGHTYELLOW_EX
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
136 elif problem['type'] == 'notice':
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
137 text_colour = colorama.Fore.LIGHTBLUE_EX
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
138 else:
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
139 text_colour = ''
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
140 ldraw_code = model.body[problem['body-index']].textual_representation()
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
141 message = str.format(
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
142 '{text_colour}{model_name}:{line_number}: {problem_type}: {message}'
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
143 '{colour_reset}\n\t{ldraw_code}',
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
144 text_colour = text_colour,
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
145 model_name = model.name,
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
146 line_number = problem['line-number'],
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
147 problem_type = problem['type'],
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
148 message = problem_text(problem, test_suite),
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
149 colour_reset = colorama.Fore.RESET,
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
150 ldraw_code = ldraw_code,
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
151 )
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
152 messages.append(message)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
153 return '\n'.join(messages)
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
154
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
155 if __name__ == '__main__':
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
156 from pprint import pprint
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
157 pprint(load_tests())

mercurial