unittest.py

Thu, 26 Aug 2021 22:04:33 +0300

author
Teemu Piippo <teemu@hecknology.net>
date
Thu, 26 Aug 2021 22:04:33 +0300
changeset 152
5e347a96869a
parent 150
fcc07f6907a8
permissions
-rwxr-xr-x

removed some alias tests that aren't mandated by the official header specification

117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
1 #!/usr/bin/env python3
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
2 from ldcheck import appname, version, version_string
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
3 from ldcheck import script_directory
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
4 from pathlib import Path
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
5 from parse import read_ldraw
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
6 from testsuite import load_tests, check_model, problem_text, all_problem_type_names
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
7
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
8 def unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
9 ''' Yields unit test paths '''
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
10 import os
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
11 unit_test_dir = Path(script_directory) / 'unittests'
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
12 for dirpath, dirnames, filenames in os.walk(unit_test_dir):
120
11af938d6076 sort unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 117
diff changeset
13 yield from sorted(
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
14 Path(dirpath) / filename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
15 for filename in filenames
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
16 if filename.endswith('.test')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
17 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
18
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
19 def parse_expectation(text):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
20 problem_type, line_no_str = str.split(text, ':')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
21 return problem_type, int(line_no_str)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
22
145
fde18c4d6784 refactoring: moved context-dependant data to new class LDrawContext. ldcheck no longer writes the config file, and looks for it in sensible locations like ~/.config and /etc. LDraw libraries can now be specified on the command line.
Teemu Piippo <teemu@hecknology.net>
parents: 127
diff changeset
23 def load_unit_test(unit_test_path, *, name, context):
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
24 with open(unit_test_path, 'rb') as device:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
25 import re
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
26 problem_types = set()
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
27 expecting = None
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
28 while True:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
29 pos = device.tell()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
30 line = bytes.decode(device.readline())
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
31 if not line:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
32 raise ValueError('unit test ended unexpectedly')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
33 match = re.match('^0 Testing: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
34 if match:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
35 set.update(problem_types, match.group(1).split())
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
36 elif str.strip(line) == '0 Expecting: none':
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
37 expecting = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
38 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
39 match = re.match('^0 Expecting: (.+)', line)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
40 if match:
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
41 if not expecting:
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
42 expecting = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
43 set.update(expecting, map(parse_expectation, match.group(1).split()))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
44 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
45 device.seek(pos)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
46 break
126
16dae12ac0f0 added bad line ending test
Teemu Piippo <teemu@hecknology.net>
parents: 123
diff changeset
47 if not problem_types or expecting is None:
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
48 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
49 'Unit test {name} does not have a proper manifest',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
50 name = name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
51 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
52 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
53 'problem_types': problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
54 'expecting': expecting,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
55 'model': read_ldraw(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
56 device,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
57 name = name,
145
fde18c4d6784 refactoring: moved context-dependant data to new class LDrawContext. ldcheck no longer writes the config file, and looks for it in sensible locations like ~/.config and /etc. LDraw libraries can now be specified on the command line.
Teemu Piippo <teemu@hecknology.net>
parents: 127
diff changeset
58 context = context
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
59 ),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
60 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
61
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
62 def parse_problem(problem):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
63 return problem.problem_class.name, problem.line_number
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
64
145
fde18c4d6784 refactoring: moved context-dependant data to new class LDrawContext. ldcheck no longer writes the config file, and looks for it in sensible locations like ~/.config and /etc. LDraw libraries can now be specified on the command line.
Teemu Piippo <teemu@hecknology.net>
parents: 127
diff changeset
65 def run_unit_test(unit_test_path, *, context, test_suite):
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
66 from os.path import basename
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
67 unit_test = load_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
68 unit_test_path,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
69 name = basename(unit_test_path),
145
fde18c4d6784 refactoring: moved context-dependant data to new class LDrawContext. ldcheck no longer writes the config file, and looks for it in sensible locations like ~/.config and /etc. LDraw libraries can now be specified on the command line.
Teemu Piippo <teemu@hecknology.net>
parents: 127
diff changeset
70 context = context,
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
71 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
72 bad_problems = set.difference(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
73 unit_test['problem_types'],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
74 all_problem_type_names(test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
75 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
76 if bad_problems:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
77 raise ValueError(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
78 'unknown problem type: {names}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
79 names = ' '.join(sorted(bad_problems))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
80 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
81 problem_types = unit_test['problem_types']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
82 report = check_model(unit_test['model'], test_suite)
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
83 expected_problems = unit_test['expecting']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
84 problems = set(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
85 filter(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
86 lambda problem: problem[0] in problem_types,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
87 map(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
88 parse_problem,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
89 report['problems']
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
90 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
91 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
92 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
93 return {
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
94 'passed': problems == expected_problems,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
95 'unexpected': set.difference(problems, expected_problems),
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
96 'missing': set.difference(expected_problems, problems),
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
97 'problem_types': problem_types,
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
98 }
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
99
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
100 def format_problem_tuple(problem_tuple):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
101 return problem_tuple[0] + ':' + str(problem_tuple[1])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
102
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
103 def run_unit_test_suite():
150
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
104 import sys
127
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
105 from argparse import ArgumentParser
150
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
106 from ldcheck import load_rcfile, LDrawContext, postprocess_library_paths
127
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
107 parser = ArgumentParser()
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
108 parser.add_argument('-d', '--debug', action = 'store_true')
150
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
109 parser.add_argument('-l', '--library', action = 'append')
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
110 parser.add_argument('--color', action = 'store_true')
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
111 args = parser.parse_args(load_rcfile() + sys.argv[1:])
fcc07f6907a8 Fix merge issues regarding the unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 145
diff changeset
112 context = LDrawContext(libraries = postprocess_library_paths(args.library))
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
113 test_suite = load_tests()
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
114 num_tested = 0
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
115 num_passed = 0
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
116 all_problem_types = all_problem_type_names(test_suite)
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
117 problem_types_tested = set()
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
118 print('Running unit test suite.')
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
119 for unit_test_path in unit_test_discovery():
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
120 try:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
121 unit_test_report = run_unit_test(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
122 unit_test_path,
145
fde18c4d6784 refactoring: moved context-dependant data to new class LDrawContext. ldcheck no longer writes the config file, and looks for it in sensible locations like ~/.config and /etc. LDraw libraries can now be specified on the command line.
Teemu Piippo <teemu@hecknology.net>
parents: 127
diff changeset
123 context = context,
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
124 test_suite = test_suite
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
125 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
126 except Exception as error:
127
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
127 if args.debug:
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
128 raise
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
129 else:
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
130 print(str.format(
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
131 'Error running {test_name}: {error}',
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
132 test_name = unit_test_path.name,
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
133 error = str(error),
97de6058109e added bfc declaration tests
Teemu Piippo <teemu@hecknology.net>
parents: 126
diff changeset
134 ))
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
135 else:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
136 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
137 '{name}: {verdict}',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
138 name = unit_test_path.name,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
139 verdict = ('FAILED', 'PASSED')[unit_test_report['passed']],
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
140 ))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
141 num_tested += 1
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
142 num_passed += int(unit_test_report['passed'])
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
143 set.update(problem_types_tested, unit_test_report['problem_types'])
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
144 if not unit_test_report['passed']:
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
145 def format_problem_list(key):
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
146 return str.join(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
147 ' ',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
148 map(format_problem_tuple, unit_test_report[key])
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
149 )
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
150 print('\tunexpected:', format_problem_list('unexpected'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
151 print('\tmissing:', format_problem_list('missing'))
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
152 print(str.format(
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
153 '{num_tested} tests run, {num_passed} tests passed.',
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
154 num_tested = num_tested,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
155 num_passed = num_passed,
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
156 ))
123
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
157 untested_problem_types = set.difference(all_problem_types, problem_types_tested)
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
158 if untested_problem_types:
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
159 print('The following problem types were not tested:')
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
160 for problem_type in sorted(untested_problem_types):
0557709c25ec added a todo list for unit tests
Teemu Piippo <teemu@hecknology.net>
parents: 122
diff changeset
161 print('\t' + problem_type)
117
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
162 if __name__ == '__main__':
178d6e54694f added unit testing
Teemu Piippo <teemu@hecknology.net>
parents:
diff changeset
163 run_unit_test_suite()

mercurial