<commit_before>#!/bin/bash

# http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail

MYNAME=$(basename $(readlink -f $0))

toaddr() {
    if [ $(whoami) == 'userrig' ]; then
        echo "<EMAIL>";
    else
        echo "$(whoami)<EMAIL>";
    fi
}

usage() {
    echo "$MYNAME: run all pipeline tests"
    echo " -d: Run dry-run tests"
    echo " -r: Run real-run tests"
}

skip_dry_runs=1
skip_real_runs=1
while getopts "dr" opt; do
    case $opt in
        d)
            skip_dry_runs=0
            ;;
        r)
            skip_real_runs=0
            ;;
        \?)
            usage
            exit 1
            ;;
    esac
done

args=""
if [ $skip_dry_runs -ne 1 ]; then
    args="$args -d"
fi
if [ $skip_real_runs -ne 1 ]; then
    args="$args -r"
fi
#echo "DEBUG args=$args" 1>&2

cd $(dirname $0)
commit=$(git describe --always --dirty)

for sh in $(find . -maxdepth 2 -mindepth 2 -name tests.sh); do
    echo "------------------------------------------------------------"
    echo "Running $sh"
    echo "------------------------------------------------------------"
    bash $sh $args
    if [ $? -ne 0 ]; then
        echo "ERROR: Tests failed"
    else
        echo "OK: Tests passed"
    fi
    echo "------------------------------------------------------------"
    echo "Running static code checks in $(dirname $sh)"
    echo "------------------------------------------------------------"
    # only warn
    set +e
    for f in $(find $(dirname $sh) -maxdepth 1 -name \*py -type f); do
        echo "Checking $f"
        PYTHONPATH=$(dirname $MYNAME)/lib pylint -j 2 -E --rcfile pylintrc $f
    done
    set -e<commit_msg>Now ignoring flymake files in pylint tests<commit_after>#!/bin/bash

# http://redsymbol.net/articles/unofficial-bash-strict-mode/
set -euo pipefail

MYNAME=$(basename $(readlink -f $0))

toaddr() {
    if [ $(whoami) == 'userrig' ]; then
        echo "<EMAIL>";
    else
        echo "$(<EMAIL>)<EMAIL>";
    fi
}

usage() {
    echo "$MYNAME: run all pipeline tests"
    echo " -d: Run dry-run tests"
    echo " -r: Run real-run tests"
}

skip_dry_runs=1
skip_real_runs=1
while getopts "dr" opt; do
    case $opt in
        d)
            skip_dry_runs=0
            ;;
        r)
            skip_real_runs=0
            ;;
        \?)
            usage
            exit 1
            ;;
    esac
done

args=""
if [ $skip_dry_runs -ne 1 ]; then
    args="$args -d"
fi
if [ $skip_real_runs -ne 1 ]; then
    args="$args -r"
fi
#echo "DEBUG args=$args" 1>&2

cd $(dirname $0)
commit=$(git describe --always --dirty)

for sh in $(find . -maxdepth 2 -mindepth 2 -name tests.sh); do
    echo "------------------------------------------------------------"
    echo "Running $sh"
    echo "------------------------------------------------------------"
    bash $sh $args
    if [ $? -ne 0 ]; then
        echo "ERROR: Tests failed"
    else
        echo "OK: Tests passed"
    fi
    echo "------------------------------------------------------------"
    echo "Running static code checks in $(dirname $sh)"
    echo "------------------------------------------------------------"
    # only warn
    set +e
    for f in $(find $(dirname $sh) -maxdepth 1 -name \*py -type f | grep -v flymake); do
        echo "Checking $f"
        PYTHONPATH=$(dirname $MYNAME)/lib pylint -j 2 -E --rcfile pylintrc $f
    done
    set -e
{'fraction_non_alphanumeric': 0.2637492607924305, 'fraction_numerical': 0.009461856889414548, 'mean_word_length': 1.2951153324287652, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '559374', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1034, 'n_words': 326}