<commit_before>import os
import re
import sys
import unittest
from glob import glob

if __name__ == "__main__":
    def get_tests():
        start_dir = os.path.join(os.path.dirname(__file__), "test")
        return unittest.TestLoader().discover(start_dir, pattern="test_to*.py")

    libdirs = glob('build/lib.*')
    if len(libdirs) > 0:
        p = [os.path.abspath(os.path.join(
            os.path.dirname(sys.argv[0]), libdirs[0])<commit_msg>fixed to run all tests again<commit_after>import os
import re
import sys
import unittest
from glob import glob

if __name__ == "__main__":
    def get_tests():
        start_dir = os.path.join(os.path.dirname(__file__), "test")
        return unittest.TestLoader().discover(start_dir, pattern="test_*.py")

    libdirs = glob('build/lib.*')
    if len(libdirs) > 0:
        p = [os.path.abspath(os.path.join(
            os.path.dirname(sys.argv[0]), libdirs[0])
{'fraction_non_alphanumeric': 0.14489571899012074, 'fraction_numerical': 0.006586169045005488, 'mean_word_length': 3.342857142857143, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2914911', 'n_tokens_mistral': 319, 'n_tokens_neox': 316, 'n_words': 68}