<commit_before># pre-commit-hook.py
# This runs all tests. Tests are dividing into those that should succeed
# (test/pass/*) and tests that check for errors (test/fail/*)

# A successful test should not print anything.

# Tests that expect a specific error message should put that message at the
# top, in a comment block. This allows the tester to check that a given error
# message is correct.

import os, subprocess, sys, signal

pass_entries = []
fail_entries = []
MODE_PASS = 1
MODE_FAIL = 0

pass_count = 0
error_count = 0
crash_count = 0

def load_filenames(path):
    ret = {}
    for (dirpath, dirnames, filenames) in os.walk(path):
        for name in filenames:
            path = dirpath + '/'
            try:
                ret[path].append(name)
            except:
                ret[path] = [name]

    return ret

def get_expected_str(filename):
    f = open(filename, "r")
    output = ''
    collecting = 0
    for line in f:
        if line.startswith('#[') and line.rstrip('\r\n') == '#[':
            collecting = True
        elif line.startswith(']#') and line.rstrip('\r\n') == "]#":
            collecting = False
        elif collecting:
            output += line.rstrip("\r\n") + "\n"
    return output

def basic_run(filename, basedir, mode, run_range):
    global pass_count, error_count, crash_count
    should_pass = (mode == MODE_PASS)
    mode_str = ""
    if mode == MODE_PASS:
        mode_str = "pass"
    else:
        mode_str = "fail"

    path = basedir + filename
    command = "./lily %s" % (path)
    subp = subprocess.Popen([command], stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, shell=True)
    (subp_stdout, subp_stderr) = subp.communicate()
    subp.wait()

    did_pass = True
    expected_stderr = get_expected_str(path)

    try:
        if subp_stderr[-1] == '\n':
            subp_stderr[:-1]
    except IndexError:
        pass

    # This makes it so that different tests don't have to reference the
    # directory path in the error (which is tedious and annoying).
    subp_stderr = subp_stderr.replace(basedir, "");

    if  len(subp_stdout) or \
        subp_stderr != expected_stderr or \
        subp.returncode == -signal.SIGSEGV:

        message = ""
        if subp.returncode == -signal.SIGSEGV:
            message = "!!!CRASHED!!!"
            crash_count += 1
        else:
            message = "!!!FAILED!!!"
            error_count += 1
<commit_msg>pre-commit-hook.py: Okay, use -g 2 now for all tests. Fixes #91.<commit_after># pre-commit-hook.py
# This runs all tests. Tests are dividing into those that should succeed
# (test/pass/*) and tests that check for errors (test/fail/*)

# A successful test should not print anything.

# Tests that expect a specific error message should put that message at the
# top, in a comment block. This allows the tester to check that a given error
# message is correct.

import os, subprocess, sys, signal

pass_entries = []
fail_entries = []
MODE_PASS = 1
MODE_FAIL = 0

pass_count = 0
error_count = 0
crash_count = 0

def load_filenames(path):
    ret = {}
    for (dirpath, dirnames, filenames) in os.walk(path):
        for name in filenames:
            path = dirpath + '/'
            try:
                ret[path].append(name)
            except:
                ret[path] = [name]

    return ret

def get_expected_str(filename):
    f = open(filename, "r")
    output = ''
    collecting = 0
    for line in f:
        if line.startswith('#[') and line.rstrip('\r\n') == '#[':
            collecting = True
        elif line.startswith(']#') and line.rstrip('\r\n') == "]#":
            collecting = False
        elif collecting:
            output += line.rstrip("\r\n") + "\n"
    return output

def basic_run(filename, basedir, mode, run_range):
    global pass_count, error_count, crash_count
    should_pass = (mode == MODE_PASS)
    mode_str = ""
    if mode == MODE_PASS:
        mode_str = "pass"
    else:
        mode_str = "fail"

    path = basedir + filename
    # The -g 2 is so that Lily's garbage collector will only allow 2
    # objects at the same time before doing a gc sweep.
    # This ensures that tests aren't passing only because the gc is not
    # being triggered.
    command = "./lily -g 2 %s" % (path)
    subp = subprocess.Popen([command], stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, shell=True)
    (subp_stdout, subp_stderr) = subp.communicate()
    subp.wait()

    did_pass = True
    expected_stderr = get_expected_str(path)

    try:
        if subp_stderr[-1] == '\n':
            subp_stderr[:-1]
    except IndexError:
        pass

    # This makes it so that different tests don't have to reference the
    # directory path in the error (which is tedious and annoying).
    subp_stderr = subp_stderr.replace(basedir, "");

    if  len(subp_stdout) or \
        subp_stderr != expected_stderr or \
        subp.returncode == -signal.SIGSEGV:

        message = ""
        if subp.returncode == -signal.SIGSEGV:
            message = "!!!CRASHED!!!"
            crash_count += 1
        else:
            message = "!!!FAILED!!!"
            error_count += 1

{'fraction_non_alphanumeric': 0.11384794414274632, 'fraction_numerical': 0.00504266873545384, 'mean_word_length': 2.3313953488372094, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864011', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1597, 'n_words': 497}