<commit_before>"""Run all unittests.

Usage:
  python3 runtests.py [-v] [-q] [pattern] ...

Where:
  -v: verbose
  -q: quiet
  pattern: optional regex patterns to match test ids (default all tests)

Note that the test id is the fully qualified name of the test,
including package, module, class and method,
e.g. 'tests.events_test.PolicyTests.testPolicy'.
"""

# Originally written by <NAME> (for NDB).

import logging
import os
import re
import sys
import unittest
import importlib.machinery

assert sys.version >= '3.3', 'Please use Python 3.3 or higher.'

TESTS_DIR = os.path.join(os.path.dirname(__file__), 'tests')


def load_tests(includes=(), excludes=()):
    test_mods = [(f[:-3], f) for f in os.listdir(TESTS_DIR)
                 if f.endswith('_test.py')]

    mods = []
    for mod, sourcefile in test_mods:
        try:
            loader = importlib.machinery.SourceFileLoader(
                mod, os.path.join(TESTS_DIR, sourcefile))
            mods.append(loader.load_module())
        except ImportError:
            pass

    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    for mod in mods:
        for name in set(dir(mod)):
            if name.endswith('Tests'):
                test_module = getattr(mod, name)
                tests = loader.loadTestsFromTestCase(test_module)
                if includes:
                    tests = [test
                             for test in tests
                             if any(re.search(pat, test.id())
                                    for pat in includes)]
                if excludes:
                    tests = [test
                             for test in tests
                             if not any(re.search(pat, test.id())
                                        for pat in excludes)]
                suite.addTests(tests)

    return suite


def main():
    excludes = []
    includes = []
    patterns = includes  # A reference.
    v = 1
    for arg in sys.argv[1:]:
        if arg.startswith('-v'):
            v += arg.count('v')
        elif arg == '-q':
            v = 0
        elif arg == '-x':
            if patterns is includes:
                patterns = excludes
            else:
                patterns = includes
        elif arg and not arg.startswith('-'):
            patterns.append(arg)
    tests = load_tests(includes, excludes)
    logger = logging.getLogger()
    if v == 0:
        logger.setLevel(logging.CRITICAL)
    elif v == 1:
        logger.setLevel(logging.ERROR)
    elif v == 2:
        logger.setLevel(logging.WARNING)
    elif v == 3:
        logger.setLevel(logging.INFO)
    elif v >= 4:
        logger.setLevel(logging.DEBUG)
    result = unittest.TextTestRunner(verbosity=v).run(tests)
    sys.exit(not result.wasSuccessful())


if __name__ == '__main__':
    main()
<commit_msg>runtests improvements.<commit_after>"""Run all unittests.

Usage:
  python3 runtests.py [-v] [-q] [pattern] ...

Where:
  -v: verbose
  -q: quiet
  pattern: optional regex patterns to match test ids (default all tests)

Note that the test id is the fully qualified name of the test,
including package, module, class and method,
e.g. 'tests.events_test.PolicyTests.testPolicy'.
"""

# Originally written by <NAME> (for NDB).

import logging
import os
import re
import sys
import unittest
import importlib.machinery

assert sys.version >= '3.3', 'Please use Python 3.3 or higher.'

TESTS_DIR = 'tests'


def load_tests(includes=(), excludes=()):
    test_mods = [(f[:-3], f) for f in os.listdir(TESTS_DIR)
                 if f.endswith('_test.py') and not f.startswith('.')]

    mods = []
    for mod, sourcefile in test_mods:
        try:
            loader = importlib.machinery.SourceFileLoader(
                mod, os.path.join(TESTS_DIR, sourcefile))
            mods.append(loader.load_module())
        except ImportError:
            pass

    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    for mod in mods:
        for name in set(dir(mod)):
            if name.endswith('Tests'):
                test_module = getattr(mod, name)
                tests = loader.loadTestsFromTestCase(test_module)
                if includes:
                    tests = [test
                             for test in tests
                             if any(re.search(pat, test.id())
                                    for pat in includes)]
                if excludes:
                    tests = [test
                             for test in tests
                             if not any(re.search(pat, test.id())
                                        for pat in excludes)]
                suite.addTests(tests)

    return suite


def main():
    excludes = []
    includes = []
    patterns = includes  # A reference.
    v = 1
    for arg in sys.argv[1:]:
        if arg.startswith('-v'):
            v += arg.count('v')
        elif arg == '-q':
            v = 0
        elif arg == '-x':
            if patterns is includes:
                patterns = excludes
            else:
                patterns = includes
        elif arg and not arg.startswith('-'):
            patterns.append(arg)
    tests = load_tests(includes, excludes)
    logger = logging.getLogger()
    if v == 0:
        logger.setLevel(logging.CRITICAL)
    elif v == 1:
        logger.setLevel(logging.ERROR)
    elif v == 2:
        logger.setLevel(logging.WARNING)
    elif v == 3:
        logger.setLevel(logging.INFO)
    elif v >= 4:
        logger.setLevel(logging.DEBUG)
    result = unittest.TextTestRunner(verbosity=v).run(tests)
    sys.exit(not result.wasSuccessful())


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.1079646017699115, 'fraction_numerical': 0.00495575221238938, 'mean_word_length': 1.733913884857281, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2912151', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1605, 'n_words': 465}