<commit_before># Convenience script to run all TKP unit tests.

import unittest
try:
    unittest.TestCase.assertIsInstance
except AttributeError:
    import unittest2 as unittest
import sys
import logging
import tkp.tests

# Suppress annoying warning messages -- we'll catch errors
# due to test failures rather than log messages.
logging.basicConfig(level=logging.CRITICAL)

args = ['tkp.tests.' + arg for arg in sys.argv[1:]]
if args:
    testfiles = [testfile for testfile in tkp.tests.testfiles if testfile in args]
else:
    testfiles = tkp.tests.testfiles

# If we can't import a particular module, we should warn the user and move on
# with the test suite, rather than bailing out completely.
for test in testfiles:
    try:
        __import__(test)
    except ImportError, e:
        print >>sys.stderr, "WARNING: Not running %s: required module failed to import (%s)" % (test, e)
        testfiles.remove(test)

result = unittest.TextTestRunner(verbosity=2).run(
    unittest.TestLoader().loadTestsFromNames(testfiles)
    )
if result.wasSuccessful():
    sys.exit(0)
else:
    sys.exit(1)<commit_msg>Return a failure if we have skipped a test due to missing dependencies<commit_after># Convenience script to run TKP unit tests.

import unittest
try:
    unittest.TestCase.assertIsInstance
except AttributeError:
    import unittest2 as unittest
import sys
import logging
import tkp.tests

# Suppress annoying warning messages -- we'll catch errors
# due to test failures rather than log messages.
logging.basicConfig(level=logging.CRITICAL)

args = ['tkp.tests.' + arg for arg in sys.argv[1:]]
if args:
    testfiles = [testfile for testfile in tkp.tests.testfiles if testfile in args]
else:
    testfiles = tkp.tests.testfiles

# If we can't import a particular module, we should warn the user and move on
# with the test suite, rather than bailing out completely. However, we log
# this as a failure.
exit_on_success = 0
for test in testfiles:
    try:
        __import__(test)
    except ImportError, e:
        print >>sys.stderr, "ERROR: Not running %s: required module failed to import (%s)" % (test, e)
        testfiles.remove(test)
        # return non-zero if we have skipped a testfile
        exit_on_success = 1

result = unittest.TextTestRunner(verbosity=2).run(
    unittest.TestLoader().loadTestsFromNames(testfiles)
    )

if result.wasSuccessful():
    sys.exit(exit_on_success)
else:
    sys.exit(1)
{'fraction_non_alphanumeric': 0.08439687114038699, 'fraction_numerical': 0.004528612597776863, 'mean_word_length': 3.8214285714285716, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6560424', 'n_tokens_mistral': 739, 'n_tokens_neox': 721, 'n_words': 267}