unittest.py

Fri, 18 Sep 2020 20:26:05 +0300

author
Teemu Piippo <teemu@hecknology.net>
date
Fri, 18 Sep 2020 20:26:05 +0300
changeset 120
11af938d6076
parent 117
178d6e54694f
child 122
116a81996832
permissions
-rwxr-xr-x

sort unit tests

117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
1 #!/usr/bin/env python3
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
2 from ldcheck import appname, version, version_string
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
3 from ldcheck import load_config, load_colours, find_ldconfig_ldr_paths
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
4 from ldcheck import script_directory
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
5 from pathlib import Path
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
6 from parse import read_ldraw
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
7 from testsuite import load_tests, check_model, problem_text, all_problem_type_names
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
8
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
9 def unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
10 ''' Yields unit test paths '''
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
11 import os
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
12 unit_test_dir = Path(script_directory) / 'unittests'
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
13 for dirpath, dirnames, filenames in os.walk(unit_test_dir):
120
11af938d6076 sort unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 117
diff changeset
14 yield from sorted(
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
15 Path(dirpath) / filename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
16 for filename in filenames
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
17 if filename.endswith('.test')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
18 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
19
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
20 def parse_expectation(text):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
21 problem_type, line_no_str = str.split(text, ':')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
22 return problem_type, int(line_no_str)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
23
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
24 def load_unit_test(unit_test_path, *, name, ldraw_directories):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
25 with open(unit_test_path, 'rb') as device:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
26 import re
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
27 problem_types = set()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
28 expecting = set()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
29 while True:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
30 pos = device.tell()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
31 line = bytes.decode(device.readline())
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
32 if not line:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
33 raise ValueError('unit test ended unexpectedly')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
34 match = re.match('^0 Testing: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
35 if match:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
36 set.update(problem_types, match.group(1).split())
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
37 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
38 match = re.match('^0 Expecting: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
39 if match:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
40 set.update(expecting, map(parse_expectation, match.group(1).split()))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
41 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
42 device.seek(pos)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
43 break
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
44 if not problem_types or not expecting:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
45 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
46 'Unit test {name} does not have a proper manifest',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
47 name = name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
48 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
49 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
50 'problem_types': problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
51 'expecting': expecting,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
52 'model': read_ldraw(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
53 device,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
54 name = name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
55 ldraw_directories = ldraw_directories
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
56 ),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
57 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
58
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
59 def parse_problem(problem):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
60 return problem.problem_class.name, problem.line_number
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
61
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
62 def run_unit_test(unit_test_path, *, config, test_suite):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
63 from os.path import basename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
64 unit_test = load_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
65 unit_test_path,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
66 name = basename(unit_test_path),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
67 ldraw_directories = config['libraries'],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
68 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
69 bad_problems = set.difference(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
70 unit_test['problem_types'],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
71 all_problem_type_names(test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
72 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
73 if bad_problems:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
74 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
75 'unknown problem type: {names}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
76 names = ' '.join(sorted(bad_problems))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
77 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
78 problem_types = unit_test['problem_types']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
79 report = check_model(unit_test['model'], test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
80 expected_problems = unit_test['expecting']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
81 problems = set(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
82 filter(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
83 lambda problem: problem[0] in problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
84 map(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
85 parse_problem,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
86 report['problems']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
87 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
88 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
89 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
90 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
91 'passed': problems == expected_problems,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
92 'unexpected': set.difference(problems, expected_problems),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
93 'missing': set.difference(expected_problems, problems),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
94 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
95
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
96 def format_problem_tuple(problem_tuple):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
97 return problem_tuple[0] + ':' + str(problem_tuple[1])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
98
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
99 def run_unit_test_suite():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
100 config = load_config()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
101 test_suite = load_tests()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
102 num_tested = 0
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
103 num_passed = 0
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
104 print('Running unit test suite.')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
105 for unit_test_path in unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
106 try:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
107 unit_test_report = run_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
108 unit_test_path,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
109 config = config,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
110 test_suite = test_suite
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
111 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
112 except Exception as error:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
113 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
114 'Error running {test_name}: {error}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
115 test_name = unit_test_path.name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
116 error = str(error),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
117 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
118 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
119 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
120 '{name}: {verdict}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
121 name = unit_test_path.name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
122 verdict = ('FAILED', 'PASSED')[unit_test_report['passed']],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
123 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
124 num_tested += 1
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
125 num_passed += int(unit_test_report['passed'])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
126 if not unit_test_report['passed']:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
127 def format_problem_list(key):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
128 return str.join(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
129 ' ',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
130 map(format_problem_tuple, unit_test_report[key])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
131 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
132 print('\tunexpected:', format_problem_list('unexpected'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
133 print('\tmissing:', format_problem_list('missing'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
134 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
135 '{num_tested} tests run, {num_passed} tests passed.',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
136 num_tested = num_tested,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
137 num_passed = num_passed,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
138 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
139 if __name__ == '__main__':
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
140 run_unit_test_suite()

mercurial