<commit_before>#!/bin/bash
MD=$(which md5sum)
if [ "$MD" == "" ]; then
    MD=$(which md5)
fi

if [ "$MD" == "" ]; then
    echo "Missing MD5!"
    exit 1
fi
VERBOSE=""
if [[ "$1" == "-v" || "$1" == "-vv" || "$1" == "-vvv" ]]; then
    VERBOSE="$1"
    shift
fi

if [ "$1" == "" ]; then
    find src/test-scripts -name '*.mvp' -exec $0 $VERBOSE \{\} \;
    find src/test-scripts -name '*.mvpt' -exec $0 $VERBOSE \{\} \;
else
    mkdir -p test/run
    echo "global_foo = \"bar\"" > test/run/.cgpiperc
    echo "bar = \"baz\"" > test/run/global.incl

    touch file1 file2

    if [ "$(echo $1 | grep ".mvpt$")" != "" ]; then
        CGPIPE_HOME=test/run dist/cgpipe $VERBOSE -nolog -f $1 > .testout 2> .testerr
    else
        CGPIPE_HOME=test/run $1 > .testout 2> .testerr
    fi

    TEST=$(cat .testout | grep -v '^#' | grep -v '^$' | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' | $MD)
    GOOD=$(cat $1.good | grep -v '^#' | grep -v '^$' |sed -e 's/^[[:blank:]]*//g' | $MD)
    if [ "$TEST" != "$GOOD" ]; then
        echo "$1 ERROR"
    else
        echo "$1 OK"
    fi
    if [ "$VERBOSE" != "" ]; then
        cat .testout | grep -v '^#' | grep -v '^$'  | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' > .testout1
        cat $1.good | grep -v '^#' | grep -v '^$'  | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' > .testout2
        echo "[EXPECTED]"
        cat .testout2
        echo ""
        echo "[GOT]"
        cat .testout1
        if [ "$TEST" != "$GOOD" ]; then
            echo ""
            echo "[DIFF]"
            diff -y .testout1 .testout2
        fi
        rm .testout1 .testout2
    fi
    rm file1 file2
    rm .testout
    rm .testerr
    rm test/run/*
fi<commit_msg>test script now sets a return code<commit_after>#!/bin/bash
MD=$(which md5sum)
if [ "$MD" == "" ]; then
    MD=$(which md5)
fi

if [ "$MD" == "" ]; then
    echo "Missing MD5!"
    exit 1
fi
VERBOSE=""
if [[ "$1" == "-v" || "$1" == "-vv" || "$1" == "-vvv" ]]; then
    VERBOSE="$1"
    shift
fi

RET=0

if [ "$1" == "" ]; then
    find src/test-scripts -name '*.mvp' -exec $0 $VERBOSE \{\} \;
    find src/test-scripts -name '*.mvpt' -exec $0 $VERBOSE \{\} \;
else
    mkdir -p test/run
    echo "global_foo = \"bar\"" > test/run/.cgpiperc
    echo "bar = \"baz\"" > test/run/global.incl

    touch file1 file2

    if [ "$(echo $1 | grep ".mvpt$")" != "" ]; then
        CGPIPE_HOME=test/run dist/cgpipe $VERBOSE -nolog -f $1 > .testout 2> .testerr
    else
        CGPIPE_HOME=test/run $1 > .testout 2> .testerr
    fi

    TEST=$(cat .testout | grep -v '^#' | grep -v '^$' | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' | $MD)
    GOOD=$(cat $1.good | grep -v '^#' | grep -v '^$' |sed -e 's/^[[:blank:]]*//g' | $MD)
    if [ "$TEST" != "$GOOD" ]; then
        echo "$1 ERROR"
        RET=1
    else
        echo "$1 OK"
    fi
    if [ "$VERBOSE" != "" ]; then
        cat .testout | grep -v '^#' | grep -v '^$'  | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' > .testout1
        cat $1.good | grep -v '^#' | grep -v '^$'  | sed -e 's/^[[:blank:]]*//g' | sed -e 's/CGPIPE ERROR.*/CGPIPE ERROR/g' > .testout2
        echo "[EXPECTED]"
        cat .testout2
        echo ""
        echo "[GOT]"
        cat .testout1
        if [ "$TEST" != "$GOOD" ]; then
            echo ""
            echo "[DIFF]"
            diff -y .testout1 .testout2
        fi
        rm .testout1 .testout2
    fi
    rm file1 file2
    rm .testout
    rm .testerr
    rm test/run/*
fi

exit $RET
{'fraction_non_alphanumeric': 0.22465981671757845, 'fraction_numerical': 0.018328242154956955, 'mean_word_length': 1.8839071257005604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1472415', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1463, 'n_words': 387}