testsuite.py

from warnings import warn

def report_element(bad_object, type, error_name, args):
    return {
        'type': type,
        'object': bad_object,
        'name': error_name,
        'args': args,
    }

def warning(bad_object, error_name, *args):
    return report_element(bad_object, 'warning', error_name, args)

def error(bad_object, error_name, *args):
    return report_element(bad_object, 'error', error_name, args)

def test_discovery():
    '''
        Finds all test modules and yields their names.
    '''
    from pkgutil import walk_packages
    import tests
    yield from sorted(
        'tests.' + result.name
        for result in walk_packages(tests.__path__)
    )

def do_manifest_integrity_checks(test_suite, module):
    '''
        Runs integrity checks on a given module's manifest.
    '''
    def check_for_extra_keys():
        extra_keys = module.manifest.keys() - test_suite.keys()
        if extra_keys:
            warn(str.format(
                '{}: extra keys in manifest: {}',
                module.__name__,
                ', '.join(map(str, extra_keys))
            ))
    def check_for_manifest_duplicates():
        for key in test_suite.keys():
            duplicates = module.manifest[key].keys() & test_suite[key].keys()
            if duplicates:
                warn(str.format(
                    '{}: redefined {} in manifests: {}',
                    module.__name__,
                    key,
                    duplicates,
                ))
    check_for_extra_keys()
    check_for_manifest_duplicates()

def load_tests():
    '''
        Imports test modules and combines their manifests into a test suite.
    '''
    test_suite = {'tests': {}, 'messages': {}}
    for module_name in test_discovery():
        from importlib import import_module
        module = import_module(module_name)
        if hasattr(module, 'manifest'):
            do_manifest_integrity_checks(test_suite, module)
            # Merge the data from the manifest
            for key in module.manifest.keys() & test_suite.keys():
                test_suite[key].update(module.manifest[key])
        else:
            warn(str.format('Module {} does not have a manifest', module_name))
    return test_suite

if __name__ == '__main__':
    from pprint import pprint
    pprint(load_tests())