unittest.py

Fri, 18 Sep 2020 21:02:44 +0300

author
Teemu Piippo <teemu@hecknology.net>
date
Fri, 18 Sep 2020 21:02:44 +0300
changeset 129
3285fe600111
parent 127
97de6058109e
child 145
fde18c4d6784
permissions
-rwxr-xr-x

unknown metacommand test

117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
1 #!/usr/bin/env python3
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
2 from ldcheck import appname, version, version_string
122
116a81996832 added more tests
Teemu Piippo <teemu@hecknology.net>
parents: 120
diff changeset
3 from ldcheck import load_config, find_ldconfig_ldr_paths
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
4 from ldcheck import script_directory
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
5 from pathlib import Path
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
6 from parse import read_ldraw
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
7 from testsuite import load_tests, check_model, problem_text, all_problem_type_names
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
8
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
9 def unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
10 ''' Yields unit test paths '''
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
11 import os
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
12 unit_test_dir = Path(script_directory) / 'unittests'
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
13 for dirpath, dirnames, filenames in os.walk(unit_test_dir):
120
11af938d6076 sort unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 117
diff changeset
14 yield from sorted(
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
15 Path(dirpath) / filename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
16 for filename in filenames
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
17 if filename.endswith('.test')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
18 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
19
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
20 def parse_expectation(text):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
21 problem_type, line_no_str = str.split(text, ':')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
22 return problem_type, int(line_no_str)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
23
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
24 def load_unit_test(unit_test_path, *, name, ldraw_directories):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
25 with open(unit_test_path, 'rb') as device:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
26 import re
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
27 problem_types = set()
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
28 expecting = None
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
29 while True:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
30 pos = device.tell()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
31 line = bytes.decode(device.readline())
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
32 if not line:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
33 raise ValueError('unit test ended unexpectedly')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
34 match = re.match('^0 Testing: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
35 if match:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
36 set.update(problem_types, match.group(1).split())
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
37 elif str.strip(line) == '0 Expecting: none':
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
38 expecting = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
39 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
40 match = re.match('^0 Expecting: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
41 if match:
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
42 if not expecting:
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
43 expecting = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
44 set.update(expecting, map(parse_expectation, match.group(1).split()))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
45 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
46 device.seek(pos)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
47 break
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
48 if not problem_types or expecting is None:
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
49 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
50 'Unit test {name} does not have a proper manifest',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
51 name = name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
52 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
53 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
54 'problem_types': problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
55 'expecting': expecting,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
56 'model': read_ldraw(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
57 device,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
58 name = name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
59 ldraw_directories = ldraw_directories
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
60 ),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
61 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
62
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
63 def parse_problem(problem):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
64 return problem.problem_class.name, problem.line_number
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
65
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
66 def run_unit_test(unit_test_path, *, config, test_suite):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
67 from os.path import basename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
68 unit_test = load_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
69 unit_test_path,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
70 name = basename(unit_test_path),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
71 ldraw_directories = config['libraries'],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
72 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
73 bad_problems = set.difference(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
74 unit_test['problem_types'],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
75 all_problem_type_names(test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
76 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
77 if bad_problems:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
78 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
79 'unknown problem type: {names}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
80 names = ' '.join(sorted(bad_problems))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
81 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
82 problem_types = unit_test['problem_types']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
83 report = check_model(unit_test['model'], test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
84 expected_problems = unit_test['expecting']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
85 problems = set(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
86 filter(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
87 lambda problem: problem[0] in problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
88 map(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
89 parse_problem,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
90 report['problems']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
91 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
92 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
93 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
94 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
95 'passed': problems == expected_problems,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
96 'unexpected': set.difference(problems, expected_problems),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
97 'missing': set.difference(expected_problems, problems),
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
98 'problem_types': problem_types,
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
99 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
100
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
101 def format_problem_tuple(problem_tuple):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
102 return problem_tuple[0] + ':' + str(problem_tuple[1])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
103
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
104 def run_unit_test_suite():
127
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
105 from argparse import ArgumentParser
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
106 parser = ArgumentParser()
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
107 parser.add_argument('-d', '--debug', action = 'store_true')
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
108 args = parser.parse_args()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
109 config = load_config()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
110 test_suite = load_tests()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
111 num_tested = 0
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
112 num_passed = 0
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
113 all_problem_types = all_problem_type_names(test_suite)
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
114 problem_types_tested = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
115 print('Running unit test suite.')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
116 for unit_test_path in unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
117 try:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
118 unit_test_report = run_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
119 unit_test_path,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
120 config = config,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
121 test_suite = test_suite
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
122 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
123 except Exception as error:
127
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
124 if args.debug:
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
125 raise
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
126 else:
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
127 print(str.format(
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
128 'Error running {test_name}: {error}',
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
129 test_name = unit_test_path.name,
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
130 error = str(error),
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
131 ))
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
132 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
133 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
134 '{name}: {verdict}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
135 name = unit_test_path.name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
136 verdict = ('FAILED', 'PASSED')[unit_test_report['passed']],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
137 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
138 num_tested += 1
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
139 num_passed += int(unit_test_report['passed'])
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
140 set.update(problem_types_tested, unit_test_report['problem_types'])
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
141 if not unit_test_report['passed']:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
142 def format_problem_list(key):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
143 return str.join(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
144 ' ',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
145 map(format_problem_tuple, unit_test_report[key])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
146 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
147 print('\tunexpected:', format_problem_list('unexpected'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
148 print('\tmissing:', format_problem_list('missing'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
149 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
150 '{num_tested} tests run, {num_passed} tests passed.',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
151 num_tested = num_tested,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
152 num_passed = num_passed,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
153 ))
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
154 untested_problem_types = set.difference(all_problem_types, problem_types_tested)
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
155 if untested_problem_types:
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
156 print('The following problem types were not tested:')
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
157 for problem_type in sorted(untested_problem_types):
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
158 print('\t' + problem_type)
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
159 if __name__ == '__main__':
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
160 run_unit_test_suite()

mercurial