testsuite.py

Fri, 18 Sep 2020 19:47:42 +0300

author
Teemu Piippo <teemu@hecknology.net>
date
Fri, 18 Sep 2020 19:47:42 +0300
changeset 116
60cac583b5df
parent 99
01941a811b5a
child 117
178d6e54694f
permissions
-rw-r--r--

fixed config not being properly loaded from script directory

16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
1 from warnings import warn
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
2
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
3 class ProblemType:
64
1c0884f5506e changed severity names to be better understood
Teemu Piippo <teemu@hecknology.net>
parents: 63
diff changeset
4 severities = ['hold', 'warning'] # in descending order
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
5 def __init__(self, name, severity, message):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
6 if severity not in ProblemType.severities:
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
7 raise ValueError(str.format(
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
8 'bad severity {severity!r}',
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
9 severity = severity,
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
10 ))
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
11 self.name = name
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
12 self.severity = severity
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
13 self.message = message
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
14 def __call__(self, bad_object, **args):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
15 return Problem(
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
16 problem_class = self,
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
17 bad_object = bad_object,
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
18 **args,
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
19 )
63
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
20 def placeholder_message(self):
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
21 if callable(self.message):
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
22 import inspect
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
23 spec = inspect.getfullargspec(self.message)
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
24 args = {}
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
25 assert not spec.varargs and not spec.varkw
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
26 for argname in spec.args + spec.kwonlyargs:
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
27 args[argname] = '<' + argname.replace('_', ' ') + '>'
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
28 return self.message(**args)
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
29 else:
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
30 return self.message
13
12d4ddc4bfd8 Got the skew test working
Santeri Piippo
parents: 12
diff changeset
31
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
32 class Problem:
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
33 def __init__(self, problem_class, bad_object, **args):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
34 self.problem_class = problem_class
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
35 self.severity = problem_class.severity
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
36 self.object = bad_object
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
37 self.args = args
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
38 def __str__(self):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
39 if callable(self.problem_class.message):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
40 return self.problem_class.message(**self.args)
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
41 else:
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
42 return self.problem_class.message
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
43
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
44 def problem_type(problem_name, **args):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
45 def wrapper(function):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
46 if not hasattr(function, 'ldcheck_problem_types'):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
47 function.ldcheck_problem_types = {}
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
48 new_type = ProblemType(name = problem_name, **args)
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
49 function.ldcheck_problem_types[problem_name] = new_type
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
50 return function
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
51 return wrapper
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
52
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
53 def report_problem(problem_name, *, bad_object, **args):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
54 return {'type': problem_name, 'bad-object': bad_object, 'args': args}
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
55
37
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
56 def name_of_package(package):
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
57 if isinstance(package, tuple):
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
58 return package[1]
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
59 else:
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
60 return package.name
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
61
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
62 def test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
63 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
64 Finds all test modules and yields their names.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
65 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
66 from pkgutil import walk_packages
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
67 import tests
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
68 yield from sorted(
37
e46fa477007b Fix operation under Python 3.4
Teemu Piippo <teemu@hecknology.net>
parents: 32
diff changeset
69 'tests.' + name_of_package(result)
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
70 for result in walk_packages(tests.__path__)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
71 )
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
72
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
73 def load_tests():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
74 '''
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
75 Imports test modules and combines their manifests into a test suite.
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
76 '''
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
77 test_suite = {'tests': []}
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
78 for module_name in test_discovery():
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
79 from importlib import import_module
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
80 module = import_module(module_name)
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
81 if hasattr(module, 'manifest'):
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
82 # Merge the data from the manifest
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
83 test_suite['tests'] += module.manifest['tests']
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
84 else:
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
85 warn(str.format('Module {} does not have a manifest', module_name))
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
86 test_suite['tests'].sort(key = lambda f: f.__name__)
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
87 return test_suite
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
88
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
89 def problem_key(problem):
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
90 rank = ProblemType.severities.index(problem.severity) # sort by severity
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
91 return (rank, problem.line_number)
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
92
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
93 def build_problem(test_function, problem_params):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
94 problem_name = problem_params['type']
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
95 problem_type = test_function.ldcheck_problem_types[problem_name]
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
96 problem_object = problem_type(
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
97 bad_object = problem_params['bad-object'],
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
98 **problem_params['args'],
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
99 )
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
100 return problem_object
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
101
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
102 def check_model(model, test_suite = None):
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
103 if not test_suite:
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
104 test_suite = load_tests()
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
105 problems = []
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
106 line_numbers = {
47
4da025d0b283 added work on header check
Teemu Piippo <teemu@hecknology.net>
parents: 37
diff changeset
107 element: (i, i + 1)
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
108 for i, element in enumerate(model.body)
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
109 }
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
110 for test_function in test_suite['tests']:
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
111 for problem_params in test_function(model):
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
112 problem = build_problem(test_function, problem_params)
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
113 # add line numbers to the problem
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
114 problem.body_index, problem.line_number \
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
115 = line_numbers[problem.object]
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
116 problem.object = None
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
117 problems.append(problem)
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
118 return {
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
119 'passed': not any(
64
1c0884f5506e changed severity names to be better understood
Teemu Piippo <teemu@hecknology.net>
parents: 63
diff changeset
120 problem.severity == 'hold'
32
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
121 for problem in problems
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
122 ),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
123 'problems': sorted(problems, key = problem_key),
75f44d3063da Reworked web front, problems are now sorted by category as well as line number
Santeri Piippo
parents: 29
diff changeset
124 }
17
327da5d00360 Added code to run the test suite.
Santeri Piippo
parents: 16
diff changeset
125
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
126 def problem_text(problem, test_suite):
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
127 message = problem.problem_class.message
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
128 if callable(message):
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
129 message = message(**problem.args)
29
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
130 return message
db6ca177c6c4 added a simple web frontend
Santeri Piippo
parents: 19
diff changeset
131
63
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
132 def iterate_problems(test_suite):
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
133 for test_function in test_suite['tests']:
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
134 yield from test_function.ldcheck_problem_types.values()
63
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
135
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
136 def all_problem_types(test_suite):
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
137 return sorted(
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
138 iterate_problems(test_suite),
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
139 key = lambda problem_type: problem_type.name
8949af6a4279 added the list of issues onto the web frontend
Teemu Piippo <teemu@hecknology.net>
parents: 62
diff changeset
140 )
62
f0a6bf48b05e Problem reporting revamp, program is now aware of its problem types
Teemu Piippo <teemu@hecknology.net>
parents: 47
diff changeset
141
16
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
142 if __name__ == '__main__':
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
143 from pprint import pprint
09cc89622262 Commit work done on test loading
Santeri Piippo
parents: 13
diff changeset
144 pprint(load_tests())

mercurial