testsuite.py

Wed, 31 Jan 2018 14:54:58 +0200

author
Santeri Piippo
date
Wed, 31 Jan 2018 14:54:58 +0200
changeset 42
be50d98d3668
parent 32
75f44d3063da
child 45
7a7c237a1717
permissions
-rw-r--r--

fixed python3.6ism

16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
1 from warnings import warn
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
2
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
3 def report_element(bad_object, type, error_name, args):
12
eb74680a5e43 commit work done on test suites...
Santeri Piippo
parents:
diff changeset
4 return {
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
5 'type': type,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
6 'object': bad_object,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
7 'name': error_name,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
8 'args': args,
13
12d4ddc4bfd8 Got the skew test working
Santeri Piippo
parents: 12
diff changeset
9 }
12d4ddc4bfd8 Got the skew test working
Santeri Piippo
parents: 12
diff changeset
10
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
11 def warning(bad_object, error_name, **args):
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
12 return report_element(bad_object, 'warning', error_name, args)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
13
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
14 def error(bad_object, error_name, **args):
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
15 return report_element(bad_object, 'error', error_name, args)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
16
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
17 def notice(bad_object, error_name, **args):
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
18 return report_element(bad_object, 'notice', error_name, args)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
19
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
20 def test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
21 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
22 Finds all test modules and yields their names.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
23 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
24 from pkgutil import walk_packages
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
25 import tests
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
26 yield from sorted(
42
be50d98d3668 fixed python3.6ism
Santeri Piippo
parents: 32
diff changeset
27 'tests.' + result[1]
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
28 for result in walk_packages(tests.__path__)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
29 )
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
30
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
31 def do_manifest_integrity_checks(test_suite, module):
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
32 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
33 Runs integrity checks on a given module's manifest.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
34 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
35 def check_for_extra_keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
36 extra_keys = module.manifest.keys() - test_suite.keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
37 if extra_keys:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
38 warn(str.format(
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
39 '{}: extra keys in manifest: {}',
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
40 module.__name__,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
41 ', '.join(map(str, extra_keys))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
42 ))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
43 def check_for_manifest_duplicates():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
44 for key in test_suite.keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
45 duplicates = module.manifest[key].keys() & test_suite[key].keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
46 if duplicates:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
47 warn(str.format(
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
48 '{}: redefined {} in manifests: {}',
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
49 module.__name__,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
50 key,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
51 duplicates,
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
52 ))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
53 check_for_extra_keys()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
54 check_for_manifest_duplicates()
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
55
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
56 def load_tests():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
57 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
58 Imports test modules and combines their manifests into a test suite.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
59 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
60 test_suite = {'tests': {}, 'messages': {}}
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
61 for module_name in test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
62 from importlib import import_module
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
63 module = import_module(module_name)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
64 if hasattr(module, 'manifest'):
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
65 do_manifest_integrity_checks(test_suite, module)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
66 # Merge the data from the manifest
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
67 for key in module.manifest.keys() & test_suite.keys():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
68 test_suite[key].update(module.manifest[key])
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
69 else:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
70 warn(str.format('Module {} does not have a manifest', module_name))
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
71 return test_suite
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
72
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
73 def problem_key(problem):
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
74 problem_hierarchy = ['error', 'warning', 'notice']
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
75 return (problem_hierarchy.index(problem['type']), problem['line-number'])
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
76
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
77 def check_model(model, test_suite = None):
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
78 if not test_suite:
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
79 test_suite = load_tests()
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
80 problems = []
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
81 line_numbers = {
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
82 element: (i, i + 1 + model.body_offset)
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
83 for i, element in enumerate(model.body)
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
84 }
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
85 for test_name, test_function in test_suite['tests'].items():
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
86 for problem in test_function(model):
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
87 problem['body-index'], problem['line-number'] \
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
88 = line_numbers[problem['object']]
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
89 del problem['object']
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
90 problems.append(problem)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
91 return {
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
92 'passed': not any(
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
93 problem['type'] == 'error'
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
94 for problem in problems
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
95 ),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
96 'problems': sorted(problems, key = problem_key),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
97 }
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
98
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
99 def problem_text(problem, test_suite):
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
100 message = test_suite['messages'][problem['name']]
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
101 if callable(message):
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
102 message = message(**problem['args'])
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
103 return message
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
104
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
105 def format_report_html(report, model, test_suite):
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
106 messages = []
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
107 for problem in report['problems']:
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
108 ldraw_code = model.body[problem['body-index']].textual_representation()
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
109 message = str.format(
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
110 '<li class="{problem_type}">{model_name}:{line_number}:'
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
111 '{problem_type}: {message}<br />{ldraw_code}</li>',
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
112 model_name = model.name,
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
113 line_number = problem['line-number'],
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
114 problem_type = problem['type'],
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
115 message = problem_text(problem, test_suite),
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
116 ldraw_code = ldraw_code,
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
117 )
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
118 messages.append(message)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
119 return '\n'.join(messages)
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
120
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
121 def format_report(report, model, test_suite):
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
122 import colorama
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
123 colorama.init()
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
124 messages = []
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
125 for problem in report['problems']:
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
126 if problem['type'] == 'error':
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
127 text_colour = colorama.Fore.LIGHTRED_EX
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
128 elif problem['type'] == 'warning':
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
129 text_colour = colorama.Fore.LIGHTYELLOW_EX
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
130 elif problem['type'] == 'notice':
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
131 text_colour = colorama.Fore.LIGHTBLUE_EX
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
132 else:
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
133 text_colour = ''
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
134 ldraw_code = model.body[problem['body-index']].textual_representation()
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
135 message = str.format(
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
136 '{text_colour}{model_name}:{line_number}: {problem_type}: {message}'
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
137 '{colour_reset}\n\t{ldraw_code}',
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
138 text_colour = text_colour,
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
139 model_name = model.name,
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
140 line_number = problem['line-number'],
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
141 problem_type = problem['type'],
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
142 message = problem_text(problem, test_suite),
18
672ebc45685a made errors prettier
Santeri Piippo
parents: 17
diff changeset
143 colour_reset = colorama.Fore.RESET,
19
9169bad392c4 better handling of ldraw code
Santeri Piippo
parents: 18
diff changeset
144 ldraw_code = ldraw_code,
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
145 )
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
146 messages.append(message)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
147 return '\n'.join(messages)
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
148
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
149 if __name__ == '__main__':
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
150 from pprint import pprint
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
151 pprint(load_tests())

mercurial