<commit_before>#!/bin/bash

# run unit tests when previous bits passed
cd test/


if [ -e success.txt ];
then
    rm success.txt
fi
if [ -e failed.txt ];
then
    rm failed.txt
fi

touch success.txt
touch failed.txt

for i in `ls *.py`
do
    echo Running test $i
    sleep 1
    python2.6 $i
    if [ $? -eq 0 ]
    then
        echo $i >> success.txt
    else
        echo $i >> failed.txt
    fi
done

NFAIL=0 #ideally no unittests should fail
TOTFAIL=`wc -l failed.txt | cut -d'f' -f1`
echo "from sys import exit; exit( $TOTFAIL !=  $NFAIL )" | python
if [ $?  -eq 0 ]
then
   echo "Testing Successful!"
   exitcode=0 # success
else
   echo "Expecting $NFAIL failures, but found $TOTFAIL failures"
   echo "Too few/many failures; some negative tests may have failed"
   exitcode=255 #failed failures != $NFAIL
fi

# cleanup
rm failed.txt
rm success.txt
cd ../
exit $exitcode<commit_msg>python 2.6 unittests<commit_after>#!/bin/bash

# run unit tests when previous bits passed
cd test/


if [ -e success26.txt ];
then
    rm success26.txt
fi
if [ -e failed26.txt ];
then
    rm failed26.txt
fi

touch success26.txt
touch failed26.txt

for i in `ls *.py`
do
    echo Running test $i
    sleep 1
    python2.6 $i
    if [ $? -eq 0 ]
    then
        echo $i >> success26.txt
    else
        echo $i >> failed26.txt
    fi
done

NFAIL=0 #ideally no unittests should fail
TOTFAIL=`wc -l failed26.txt | cut -d'f' -f1`
echo "from sys import exit; exit( $TOTFAIL !=  $NFAIL )" | python
if [ $?  -eq 0 ]
then
   echo "Testing Successful!"
   exitcode=0 # success
else
   echo "Expecting $NFAIL failures, but found $TOTFAIL failures"
   echo "Too few/many failures; some negative tests may have failed"
   exitcode=255 #failed failures != $NFAIL
fi

# cleanup
rm failed26.txt
rm success26.txt
cd ../
exit $exitcode
{'fraction_non_alphanumeric': 0.10668877833056938, 'fraction_numerical': 0.02542841348811498, 'mean_word_length': 2.739669421487603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7075098', 'n_tokens_mistral': 672, 'n_tokens_neox': 635, 'n_words': 235}