<commit_before>#!/bin/bash
if [ "$#" -ne "4" ];
then
    echo ""
    echo "expected arguments: [SVNPATH] [OUTDIR] [START-REV] [END-REV]"
    echo ""
    exit
fi

SVN_ROOT=$1
OUTPUT_DIR=$2
START_REVISION=$3
END_REVISION=$4

cd $SVN_ROOT
for ((rev=$START_REVISION; rev <= $END_REVISION; rev++))
do
#   svn checkout -r $rev
#   make nightly > performance.results
   awk '/Results:/ {flag=1}flag' performance.results | tail -n +2 > performance.data
   cat performance.data | while read line;
   do
      set $line
      echo -e "$rev\t$2" >> $OUTPUT_DIR/$1.results
   done
done

<commit_msg>Fixes to regression testing script<commit_after>#!/bin/bash
if [ "$#" -ne "4" ];
then
    echo ""
    echo "expected arguments: [SVNPATH] [OUTDIR] [START-REV] [END-REV]"
    echo ""
    exit
fi

SVN_ROOT=$1
OUTPUT_DIR=$2
START_REVISION=$3
END_REVISION=$4

cd $SVN_ROOT
for ((rev=$START_REVISION; rev <= $END_REVISION; rev++))
do
   svn update -r $rev --non-interactive --force
   svn revert --recursive .
   make install
   make nightly > performance.results
   awk '/Results:/ {flag=1}flag' performance.results | tail -n +2 > performance.data
   cat performance.data | while read line;
   do
      set $line
      echo -e "$rev\t$2" >> $OUTPUT_DIR/$1.results
   done
done


{'fraction_non_alphanumeric': 0.1506740681998414, 'fraction_numerical': 0.014274385408406027, 'mean_word_length': 3.2779661016949153, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645720', 'n_tokens_mistral': 484, 'n_tokens_neox': 470, 'n_words': 125}