<commit_before>#!/bin/bash -e

set -o pipefail

COMPILE_INPUT=$(mktemp)
COMPILE_OUTPUT=$(mktemp)
QUERY_INPUT=$(mktemp)
QUERY_OUTPUT=$(mktemp)

cleanup()
{
    rm -f $TRIE $COMPILE_INPUT $COMPILE_OUTPUT $QUERY_INPUT $QUERY_OUTPUT
}

trap cleanup EXIT

compile_input()
{
    cat >$COMPILE_INPUT
}

compile_output()
{
    cat >$COMPILE_OUTPUT
}

query_input()
{
    cat >$QUERY_INPUT
}

query_output()
{
    cat >$QUERY_OUTPUT
}

runtest()
{
    ARGS=$1

    TRIE=$(mktemp)

    if [ "x${USE_VALGRIND:-yes}" = "xno" ] || ! which valgrind; then
        VALGRIND=""
    else
        VALGRIND="valgrind --error-exitcode=1 -q"
    fi

    libtool --mode=execute $VALGRIND ./list-compile $ARGS "$COMPILE_INPUT" "$TRIE" \
        | diff -q - "$COMPILE_OUTPUT"

    libtool --mode=execute $VALGRIND ./list-query $TRIE <"$QUERY_INPUT" \
        | diff -q - "$QUERY_OUTPUT"
}<commit_msg>Clean up test helper script<commit_after>#!/bin/bash -e

set -o pipefail

TRIE=$(mktemp)
COMPILE_INPUT=$(mktemp)
COMPILE_OUTPUT=$(mktemp)
QUERY_INPUT=$(mktemp)
QUERY_OUTPUT=$(mktemp)

cleanup()
{
    rm -f $TRIE $COMPILE_INPUT $COMPILE_OUTPUT $QUERY_INPUT $QUERY_OUTPUT
}

trap cleanup EXIT

compile_input()
{
    cat >$COMPILE_INPUT
}

compile_output()
{
    cat >$COMPILE_OUTPUT
}

query_input()
{
    cat >$QUERY_INPUT
}

query_output()
{
    cat >$QUERY_OUTPUT
}

runtest()
{
    ARGS=$1

    VALGRIND="valgrind --error-exitcode=1 -q"
    if [ "x${USE_VALGRIND:-yes}" = "xno" ] || ! which valgrind; then
        VALGRIND=""
    fi
    RUNNER="libtool --mode=execute $VALGRIND"

    $RUNNER ./list-compile $ARGS $COMPILE_INPUT $TRIE \
        | diff -q - "$COMPILE_OUTPUT"

    $RUNNER ./list-query $TRIE <$QUERY_INPUT \
        | diff -q - "$QUERY_OUTPUT"
}
{'fraction_non_alphanumeric': 0.15380184331797234, 'fraction_numerical': 0.002304147465437788, 'mean_word_length': 3.2783251231527095, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1753829', 'n_tokens_mistral': 709, 'n_tokens_neox': 701, 'n_words': 134}