<commit_before>
# pre-commit-hook.py
# This runs all tests. Tests are dividing into those that should succeed
# (test/pass/*) and tests that check for errors (test/fail/*)

# A successful test should not print anything.

# Tests that expect a specific error message should put that message at the
# top, in a comment block. This allows the tester to check that a given error
# message is correct.

import os, subprocess, sys, signal

pass_count = 0
error_count = 0
crash_count = 0
test_count = 0

def get_expected_str(filename):
    f = open(filename, "r")
    output = ''
    collecting = 0
    for line in f:
        if line.startswith('#[') and line.rstrip('\r\n') == '#[':
            collecting = True
        elif line.startswith(']#') and line.rstrip('\r\n') == "]#":
            collecting = False
        elif collecting:
            # This assumes that line is always \n terminated.
            output += line
    return output

def run_test(options, dirpath, filepath):
    global pass_count, error_count, crash_count, test_count

    test_count += 1

    should_pass = (options['mode'] == 'pass')
    fullpath = dirpath + filepath
    expected_stderr = get_expected_str(fullpath)

    # There are no paths with spaces, so this is ok.
    command = ("./lily %s %s" % (options['invoke'], fullpath)).split(" ")

    subp = subprocess.Popen(command, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
    (subp_stdout, subp_stderr) = subp.communicate()
    subp.wait()

    # This makes it so that different tests don't have to reference the
    # directory path in the error (which is tedious and annoying).
    subp_stderr = subp_stderr.replace(dirpath, "")
    # For Windows, this fixes newlines so the strings equal.
    subp_stderr = subp_stderr.replace("\r\n", "\n")

    if subp_stderr != expected_stderr or \
       subp.returncode == -signal.SIGSEGV:

        message = ""
        if subp.returncode == -signal.SIGSEGV:
            message = "!!!CRASHED!!!"
            crash_count += 1
        else:
            message = "!!!FAILED!!!"
            error_count += 1

        sys.stdout.write("#%d test %s %s\n" % (test_count, \
                os.path.basename(filepath), message))
    else:
        pass_count += 1

def get_options_for(dirpath):
    options = {'invoke': '-gstart 2 -gmul 0'}

    # By default, the gc threshold is set to 2 so that the tests don't
    # pass only because the garbage collector isn't invoked.
    if dirpath.endswith('tag_mode' + os.sep):
        # This causes the tests run in this directory to be run in
        # tagged mode (code will be between <?lily ... ?> tags only).
        options['invoke'] += ' -t'

    options['mode'] = 'pass' if dirpath.endswith('pass') else 'fail'
    return options

def process_test_dir(basepath):
    mode = 'pass' if basepath.endswith('pass') else 'fail'

    # I use _list as a suffix to make the difference more obvious than,
    # say, filepaths versus filepath.
    for (dirpath, dirpath_list, filepath_list) in os.walk(basepath):
        dirpath += os.sep
        options = get_options_for(dirpath)
        for filepath in filepath_list:
            run_test(options, dirpath, filepath)

if __name__ == '__main__':
    process_test_dir('test' + os.sep + 'fail')
    process_test_dir('test' + os.sep + 'pass')

print ('Final stats: %d tests passed, %d errors, %d crashed.' \
        % (pass_count, error_count, crash_count))

if error_count == 0 and crash_count == 0:
    sys.exit(0)<commit_msg>Clean pre-commit-hook.py a little bit.<commit_after>
# pre-commit-hook.py
# This runs all tests. Tests are dividing into those that should succeed
# (test/pass/*) and tests that check for errors (test/fail/*)

# A successful test should not print anything.

# Tests that expect a specific error message should put that message at the
# top, in a comment block. This allows the tester to check that a given error
# message is correct.

import os, subprocess, sys, signal

pass_count = 0
error_count = 0
crash_count = 0
test_count = 0

def get_expected_str(filename):
    f = open(filename, "r")
    output = ''
    collecting = 0
    for line in f:
        if line.startswith('#[') and line.rstrip('\r\n') == '#[':
            collecting = True
        elif line.startswith(']#') and line.rstrip('\r\n') == "]#":
            collecting = False
        elif collecting:
            # This assumes that line is always \n terminated.
            output += line
    return output

def run_test(options, dirpath, filepath):
    global pass_count, error_count, crash_count, test_count

    test_count += 1

    fullpath = dirpath + filepath
    expected_stderr = get_expected_str(fullpath)

    # There are no paths with spaces, so this is ok.
    command = ("./lily %s %s" % (options['invoke'], fullpath)).split(" ")

    subp = subprocess.Popen(command, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
    (subp_stdout, subp_stderr) = subp.communicate()
    subp.wait()

    # This makes it so that different tests don't have to reference the
    # directory path in the error (which is tedious and annoying).
    subp_stderr = subp_stderr.replace(dirpath, "")
    # For Windows, this fixes newlines so the strings equal.
    subp_stderr = subp_stderr.replace("\r\n", "\n")

    if subp_stderr != expected_stderr or \
       subp.returncode == -signal.SIGSEGV:

        message = ""
        if subp.returncode == -signal.SIGSEGV:
            message = "!!!CRASHED!!!"
            crash_count += 1
        else:
            message = "!!!FAILED!!!"
            error_count += 1

        sys.stdout.write("#%d test %s %s\n" % (test_count, \
                os.path.basename(filepath), message))
    else:
        pass_count += 1

def get_options_for(dirpath):
    # This makes the garbage collector perform a sweep after a third object
    # tries to get a tag. The growth factor is zero, which then forces the gc
    # to continually sweep afterward.
    # This hyperaggressive behavior helps uncover bugs from forgetting to tag
    # values appropriately.
    options = {'invoke': '-gstart 2 -gmul 0'}

    if dirpath.endswith('tag_mode'):
        # This causes the tests run in this directory to be run in
        # tagged mode (code will be between <?lily ... ?> tags only).
        options['invoke'] += ' -t'

    return options

def process_test_dir(basepath):
    # I use _list as a suffix to make the difference more obvious than,
    # say, filepaths versus filepath.
    for (dirpath, dirpath_list, filepath_list) in os.walk(basepath):
        options = get_options_for(dirpath)
        dirpath += os.sep
        for filepath in filepath_list:
            run_test(options, dirpath, filepath)

if __name__ == '__main__':
    process_test_dir('test' + os.sep + 'fail')
    process_test_dir('test' + os.sep + 'pass')

print ('Final stats: %d tests passed, %d errors, %d crashed.' \
        % (pass_count, error_count, crash_count))

if error_count == 0 and crash_count == 0:
    sys.exit(0)
{'fraction_non_alphanumeric': 0.11287326760965852, 'fraction_numerical': 0.004143449064152022, 'mean_word_length': 2.755364806866953, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6393284', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2114, 'n_words': 716}