<commit_before>#!/bin/bash

set -E
trap '[ "$?" -ne 77 ] || exit 77' ERR


### until we really support makefiles better - instead of using project files to parallelize
if [ `uname -s | cut -b 1-6` == "CYGWIN" ] ; then
	if [ -z ${PARALELLMAKEFLAG} ] ; then : ${PARALELLMAKEFLAG:=-j1}; fi
fi


: ${CONTINUE:=0}
if [ -z ${INCLUDE_VALGRIND_MEMCHECK_TESTS+x} ] ; then hash valgrind 2> /dev/null; if [ $? -eq 0 ]; then  INCLUDE_VALGRIND_MEMCHECK_TESTS=1; else  INCLUDE_VALGRIND_MEMCHECK_TESTS=0; fi; fi
if [ -z ${INCLUDE_VALGRIND_HELGRIND_TESTS+x} ] ; then hash valgrind 2> /dev/null; if [ $? -eq 0 ]; then  INCLUDE_VALGRIND_HELGRIND_TESTS=1; else  INCLUDE_VALGRIND_HELGRIND_TESTS=0; fi; fi
: ${INCLUDE_PERFORMANCE_TESTS:=1}
if [ -z ${CLOBBER_FIRST+x} ] ; then if [ $CONTINUE -eq 1 ]; then  CLOBBER_FIRST=0; else  CLOBBER_FIRST=1; fi; fi
: ${PARALELLMAKEFLAG:=-j10}
: ${BUILD_CONFIGURATIONS_MAKEFILE_TARGET:=default-configurations}
: ${USE_TEST_BASENAME:=""}
BUILD_EXTRA_COMPILERS_IF_MISSING=
if [ $CONTINUE -eq 0 ]  && [ "$BUILD_CONFIGURATIONS_MAKEFILE_TARGET" == "default-configurations" ] && [ "$(uname -s)" == "Linux" ]; then
	BUILD_EXTRA_COMPILERS_IF_MISSING=1
else
	BUILD_EXTRA_COMPILERS_IF_MISSING=0
fi


VER=`ScriptsLib/ExtractVersionInformation.sh STROIKA_VERSION FullVersionString`


mkdir -p Tests/HistoricalRegressionTestResults


if [ "$(uname -s)" == "Darwin" ] ; then
	if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="MacOS_XCode10"; fi
    echo "USING MacOS($USE_TEST_BASENAME)..."
fi
if [ `uname -s | cut -b 1-6` == "CYGWIN" ] ; then
	if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="Windows_VS2k17"; fi
    echo "USING VS2k17 ($USE_TEST_BASENAME)..."
fi


RASPBERRYPI_REMOTE_MACHINE=raspberrypi
RASPBERRYPI_REMOTE_WITH_LOGIN=lewis@$RASPBERRYPI_REMOTE_MACHINE
ARMTESTMACHINEAVAIL=`(ping raspberrypi -c 4 2>/dev/null 1>/dev/null); echo $?`
if [ $ARMTESTMACHINEAVAIL -eq 0 ]; then
	ARMTESTMACHINEAVAIL=1
else
	ARMTESTMACHINEAVAIL=0
fi

if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="Linux"; fi
TEST_OUT_FILE=Tests/HistoricalRegressionTestResults/REGRESSION-TESTS-$USE_TEST_BASENAME-$VER-OUT.txt


RASPBERRYPICONFIGS=`make list-configurations TAGS="raspberrypi"`
RASPBERRYPIVALGRINDCONFIGS=`make list-configurations TAGS="raspberrypi valgrind"`

echo $RASPBERRYPIVALGRINDCONFIGS > /tmp/raspvalconfigs.txt
make list-configurations TAGS="valgrind"  > /tmp/allvalgrindconfigs.txt

#if [ $DO_ONLY_DEFAULT_CONFIGURATIONS -eq 1 ] ; then
#	INCLUDE_VALGRIND_MEMCHECK_TESTS=0
#	INCLUDE_VALGRIND_HELGRIND_TESTS=0
#	INCLUDE_PERFORMANCE_TESTS=0
#fi
if [ `uname -s | cut -b 1-6` != "CYGWIN" ] ; then
	LOCALVALGRINDCONFIGS=`comm -23 <(sort /tmp/allvalgrindconfigs.txt) <(sort /tmp/raspvalconfigs.txt)`
	if [ `echo $LOCALVALGRINDCONFIGS | wc -w` -eq 0 ] ; then
		INCLUDE_VALGRIND_MEMCHECK_TESTS=0
		INCLUDE_VALGRIND_HELGRIND_TESTS=0
	fi
else
	LOCALVALGRINDCONFIGS=""
fi



PREFIX_OUT_LABEL=")))-"

STARTAT_INT=$(date +%s)
STARTAT=`date`;



if [ $CONTINUE -ne 0 ] ; then
	echo "CONTINUING (output to $TEST_OUT_FILE) - started at $STARTAT"
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "CONTINUING (output to $TEST_OUT_FILE) - started at $STARTAT" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
else
	rm -f $TEST_OUT_FILE
	echo "Resetting all configurations to standard regression test set (output to $TEST_OUT_FILE) - started at $STARTAT"
	echo "$PREFIX_OUT_LABEL" "Resetting all configurations to standard regression test set (output to $TEST_OUT_FILE) - started at $STARTAT" >>$TEST_OUT_FILE 2>&1
fi



echo "$PREFIX_OUT_LABEL" "REGRESSION TEST CONFIGURATION VARIABLES:" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    USE_TEST_BASENAME=$USE_TEST_BASENAME" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    PARALELLMAKEFLAG=$PARALELLMAKEFLAG" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    CONTINUE=$CONTINUE" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_VALGRIND_MEMCHECK_TESTS=$INCLUDE_VALGRIND_MEMCHECK_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_VALGRIND_HELGRIND_TESTS=$INCLUDE_VALGRIND_HELGRIND_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    BUILD_EXTRA_COMPILERS_IF_MISSING=$BUILD_EXTRA_COMPILERS_IF_MISSING" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_PERFORMANCE_TESTS=$INCLUDE_PERFORMANCE_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    ARMTESTMACHINEAVAIL=$ARMTESTMACHINEAVAIL" >>$TEST_OUT_FILE 2>&1
echo >>$TEST_OUT_FILE 2>&1




if [ $BUILD_EXTRA_COMPILERS_IF_MISSING -ne 0 ] ; then
	if ! [ -e /private-compiler-builds/clang-7.0.0 ]; then
		echo -n "Building /private-compiler-builds/clang-7.0.0..."
		VERSION=7.0.0 ./ScriptsLib/BuildClang.sh  >>$TEST_OUT_FILE 2>&1
		rm -rf BUILDDIR-LLVM-7.0.0
		echo "done"
	fi
fi




if [ $CONTINUE -eq 0 ] ; then
	rm -rf ConfigurationFiles
	make $BUILD_CONFIGURATIONS_MAKEFILE_TARGET >>$TEST_OUT_FILE 2>&1
fi


NUM_CONFIGURATIONS=`ScriptsLib/GetConfigurations | wc -w`
NUM_REGTESTS=`wc -l Tests/Tests-Description.txt | awk '{print $1;}'`
NUM_PASSES_OF_REGTESTS_RUN=$NUM_CONFIGURATIONS

echo "Building configurations ($NUM_CONFIGURATIONS):"
echo "$PREFIX_OUT_LABEL" "Building configurations ($NUM_CONFIGURATIONS):" >>$TEST_OUT_FILE 2>&1
for i in `ScriptsLib/GetConfigurations`; do
	echo "   $i";
	echo "   $i"  >>$TEST_OUT_FILE 2>&1;
done

if [ $CLOBBER_FIRST -ne 0 ] ; then
  	echo "Make clobber"
  	echo "Make clobber" >>$TEST_OUT_FILE 2>&1
  	make clobber >>$TEST_OUT_FILE 2>&1
else
	echo "Skipping Clobber"
	echo "Skipping Clobber" >>$TEST_OUT_FILE 2>&1
fi

STAGE_STARTAT_INT=$(date +%s)
echo -n "Make all ($PARALELLMAKEFLAG)..."
echo "$PREFIX_OUT_LABEL" "Make all ($PARALELLMAKEFLAG)..."  >>$TEST_OUT_FILE 2>&1
make all $PARALELLMAKEFLAG >>$TEST_OUT_FILE 2>&1
STAGE_TOTAL_MINUTES_SPENT=$(($(( $(date +%s) - $STAGE_STARTAT_INT )) / 60))
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)"
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)">>$TEST_OUT_FILE 2>&1


STAGE_STARTAT_INT=$(date +%s)
echo -n "Run-Tests ALL-Local..."
echo "$PREFIX_OUT_LABEL" "Run-Tests ALL-Local..." >>$TEST_OUT_FILE 2>&1
make run-tests >>$TEST_OUT_FILE 2>&1
STAGE_TOTAL_MINUTES_SPENT=$(($(( $(date +%s) - $STAGE_STARTAT_INT )) / 60))
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)"
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)">>$TEST_OUT_FILE 2>&1


### @todo - cleanup so have better logical test
# cuz arm tests done explicitly below (maybe should automatically substract 'cross-compiled tests')
NUM_PASSES_OF_REGTESTS_RUN=$(($NUM_PASSES_OF_REGTESTS_RUN - 6))
echo -n "Run-Tests raspberrypi remote..."
echo "$PREFIX_OUT_LABEL" "Run-Tests raspberrypi remote..." >>$TEST_OUT_FILE 2>&1
if [ $ARMTESTMACHINEAVAIL ]; then
	STAGE_STARTAT_INT=$(date +%s)
	for i in $RASPBERRYPICONFIGS; do 
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN >>$TEST_OUT_FILE 2>&1
		NUM_PASSES_OF_REGTESTS_RUN=$(($NUM_PASSES_OF_REGTESTS_RUN + 1))
	done; 
	for i in $RASPBERRYPIVALGRINDCONFIGS; do 
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=memcheck" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=memcheck >>$TEST_OUT_FILE 2>&1
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=helgrind" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=helgrind >>$TEST_OUT_FILE 2>&1<commit_msg>more cleanups of regressiontest script (and output of variable results)<commit_after>#!/bin/bash

set -E
trap '[ "$?" -ne 77 ] || exit 77' ERR


### until we really support makefiles better - instead of using project files to parallelize
if [ `uname -s | cut -b 1-6` == "CYGWIN" ] ; then
	if [ -z ${PARALELLMAKEFLAG} ] ; then : ${PARALELLMAKEFLAG:=-j1}; fi
fi


: ${CONTINUE:=0}
if [ -z ${INCLUDE_VALGRIND_MEMCHECK_TESTS+x} ] ; then hash valgrind 2> /dev/null; if [ $? -eq 0 ]; then  INCLUDE_VALGRIND_MEMCHECK_TESTS=1; else  INCLUDE_VALGRIND_MEMCHECK_TESTS=0; fi; fi
if [ -z ${INCLUDE_VALGRIND_HELGRIND_TESTS+x} ] ; then hash valgrind 2> /dev/null; if [ $? -eq 0 ]; then  INCLUDE_VALGRIND_HELGRIND_TESTS=1; else  INCLUDE_VALGRIND_HELGRIND_TESTS=0; fi; fi
: ${INCLUDE_PERFORMANCE_TESTS:=1}
if [ -z ${CLOBBER_FIRST+x} ] ; then if [ $CONTINUE -eq 1 ]; then  CLOBBER_FIRST=0; else  CLOBBER_FIRST=1; fi; fi
: ${PARALELLMAKEFLAG:=-j10}
: ${BUILD_CONFIGURATIONS_MAKEFILE_TARGET:=default-configurations}
: ${USE_TEST_BASENAME:=""}
BUILD_EXTRA_COMPILERS_IF_MISSING=
if [ $CONTINUE -eq 0 ]  && [ "$BUILD_CONFIGURATIONS_MAKEFILE_TARGET" == "default-configurations" ] && [ "$(uname -s)" == "Linux" ]; then
	BUILD_EXTRA_COMPILERS_IF_MISSING=1
else
	BUILD_EXTRA_COMPILERS_IF_MISSING=0
fi


VER=`ScriptsLib/ExtractVersionInformation.sh STROIKA_VERSION FullVersionString`


mkdir -p Tests/HistoricalRegressionTestResults


if [ "$(uname -s)" == "Darwin" ] ; then
	if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="MacOS_XCode10"; fi
    echo "USING MacOS($USE_TEST_BASENAME)..."
fi
if [ `uname -s | cut -b 1-6` == "CYGWIN" ] ; then
	if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="Windows_VS2k17"; fi
    echo "USING VS2k17 ($USE_TEST_BASENAME)..."
fi


RASPBERRYPI_REMOTE_MACHINE=raspberrypi
RASPBERRYPI_REMOTE_WITH_LOGIN=lewis@$RASPBERRYPI_REMOTE_MACHINE
ARM_TEST_MACHINE_AVAIL=`(ping raspberrypi -c 4 2>/dev/null 1>/dev/null); echo $?`
if [ $ARM_TEST_MACHINE_AVAIL -eq 0 ]; then
	ARM_TEST_MACHINE_AVAIL=1
else
	ARM_TEST_MACHINE_AVAIL=0
fi

if [ "$USE_TEST_BASENAME" == "" ] ; then USE_TEST_BASENAME="Linux"; fi
TEST_OUT_FILE=Tests/HistoricalRegressionTestResults/REGRESSION-TESTS-$USE_TEST_BASENAME-$VER-OUT.txt


RASPBERRYPICONFIGS=`make list-configurations TAGS="raspberrypi"`
RASPBERRYPIVALGRINDCONFIGS=`make list-configurations TAGS="raspberrypi valgrind"`


#if [ $DO_ONLY_DEFAULT_CONFIGURATIONS -eq 1 ] ; then
#	INCLUDE_VALGRIND_MEMCHECK_TESTS=0
#	INCLUDE_VALGRIND_HELGRIND_TESTS=0
#	INCLUDE_PERFORMANCE_TESTS=0
#fi
echo $RASPBERRYPIVALGRINDCONFIGS > /tmp/raspvalconfigs.txt
make list-configurations TAGS="valgrind"  > /tmp/allvalgrindconfigs.txt
if [ `uname -s | cut -b 1-6` != "CYGWIN" ] ; then
	LOCAL_VALGRIND_CONFIGS=`comm -23 <(sort /tmp/allvalgrindconfigs.txt) <(sort /tmp/raspvalconfigs.txt)`
	if [ `echo $LOCAL_VALGRIND_CONFIGS | wc -w` -eq 0 ] ; then
		INCLUDE_VALGRIND_MEMCHECK_TESTS=0
		INCLUDE_VALGRIND_HELGRIND_TESTS=0
	fi
else
	LOCAL_VALGRIND_CONFIGS=""
fi
rm -f /tmp/raspvalconfigs.txt /tmp/allvalgrindconfigs.txt


PREFIX_OUT_LABEL=")))-"

STARTAT_INT=$(date +%s)
STARTAT=`date`;



if [ $CONTINUE -ne 0 ] ; then
	echo "CONTINUING (output to $TEST_OUT_FILE) - started at $STARTAT"
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "CONTINUING (output to $TEST_OUT_FILE) - started at $STARTAT" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
	echo "$PREFIX_OUT_LABEL" "---------------------" >>$TEST_OUT_FILE 2>&1
else
	rm -f $TEST_OUT_FILE
	echo "Resetting all configurations to standard regression test set (output to $TEST_OUT_FILE) - started at $STARTAT"
	echo "$PREFIX_OUT_LABEL" "Resetting all configurations to standard regression test set (output to $TEST_OUT_FILE) - started at $STARTAT" >>$TEST_OUT_FILE 2>&1
fi



echo "$PREFIX_OUT_LABEL" "REGRESSION TEST CONFIGURATION VARIABLES:" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    USE_TEST_BASENAME=$USE_TEST_BASENAME" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    PARALELLMAKEFLAG=$PARALELLMAKEFLAG" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    CONTINUE=$CONTINUE" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_VALGRIND_MEMCHECK_TESTS=$INCLUDE_VALGRIND_MEMCHECK_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_VALGRIND_HELGRIND_TESTS=$INCLUDE_VALGRIND_HELGRIND_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    BUILD_EXTRA_COMPILERS_IF_MISSING=$BUILD_EXTRA_COMPILERS_IF_MISSING" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    INCLUDE_PERFORMANCE_TESTS=$INCLUDE_PERFORMANCE_TESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    ARM_TEST_MACHINE_AVAIL=$ARM_TEST_MACHINE_AVAIL" >>$TEST_OUT_FILE 2>&1
if [ "$LOCAL_VALGRIND_CONFIGS" != ""]; then
	echo "$PREFIX_OUT_LABEL" "    LOCAL_VALGRIND_CONFIGS=$LOCAL_VALGRIND_CONFIGS" >>$TEST_OUT_FILE 2>&1
fi
echo >>$TEST_OUT_FILE 2>&1




if [ $BUILD_EXTRA_COMPILERS_IF_MISSING -ne 0 ] ; then
	if ! [ -e /private-compiler-builds/clang-7.0.0 ]; then
		echo -n "Building /private-compiler-builds/clang-7.0.0..."
		VERSION=7.0.0 ./ScriptsLib/BuildClang.sh  >>$TEST_OUT_FILE 2>&1
		rm -rf BUILDDIR-LLVM-7.0.0
		echo "done"
	fi
fi



### Create the actual configuration files that drive what is built
if [ $CONTINUE -eq 0 ] ; then
	rm -rf ConfigurationFiles
	make $BUILD_CONFIGURATIONS_MAKEFILE_TARGET >>$TEST_OUT_FILE 2>&1
fi


NUM_CONFIGURATIONS=`ScriptsLib/GetConfigurations | wc -w`
NUM_REGTESTS=`wc -l Tests/Tests-Description.txt | awk '{print $1;}'`
NUM_PASSES_OF_REGTESTS_RUN=$NUM_CONFIGURATIONS



echo "$PREFIX_OUT_LABEL" "REGRESSION TEST CONFIGURATION VARIABLES(UPDATES):" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    NUM_REGTESTS=$NUM_REGTESTS" >>$TEST_OUT_FILE 2>&1
echo "$PREFIX_OUT_LABEL" "    NUM_CONFIGURATIONS=$NUM_CONFIGURATIONS" >>$TEST_OUT_FILE 2>&1
echo >>$TEST_OUT_FILE 2>&1



echo "Building configurations ($NUM_CONFIGURATIONS):"
echo "$PREFIX_OUT_LABEL" "Building configurations ($NUM_CONFIGURATIONS):" >>$TEST_OUT_FILE 2>&1
for i in `ScriptsLib/GetConfigurations`; do
	echo "   $i";
	echo "   $i"  >>$TEST_OUT_FILE 2>&1;
done

if [ $CLOBBER_FIRST -ne 0 ] ; then
  	echo "Make clobber"
  	echo "Make clobber" >>$TEST_OUT_FILE 2>&1
  	make clobber >>$TEST_OUT_FILE 2>&1
else
	echo "Skipping Clobber"
	echo "Skipping Clobber" >>$TEST_OUT_FILE 2>&1
fi





STAGE_STARTAT_INT=$(date +%s)
echo -n "Make all ($PARALELLMAKEFLAG)..."
echo "$PREFIX_OUT_LABEL" "Make all ($PARALELLMAKEFLAG)..."  >>$TEST_OUT_FILE 2>&1
make all $PARALELLMAKEFLAG >>$TEST_OUT_FILE 2>&1
STAGE_TOTAL_MINUTES_SPENT=$(($(( $(date +%s) - $STAGE_STARTAT_INT )) / 60))
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)"
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)">>$TEST_OUT_FILE 2>&1


STAGE_STARTAT_INT=$(date +%s)
echo -n "Run-Tests ALL-Local..."
echo "$PREFIX_OUT_LABEL" "Run-Tests ALL-Local..." >>$TEST_OUT_FILE 2>&1
make run-tests >>$TEST_OUT_FILE 2>&1
STAGE_TOTAL_MINUTES_SPENT=$(($(( $(date +%s) - $STAGE_STARTAT_INT )) / 60))
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)"
echo "done (in $STAGE_TOTAL_MINUTES_SPENT minutes)">>$TEST_OUT_FILE 2>&1


### @todo - cleanup so have better logical test
# cuz arm tests done explicitly below (maybe should automatically substract 'cross-compiled tests')
NUM_PASSES_OF_REGTESTS_RUN=$(($NUM_PASSES_OF_REGTESTS_RUN - 6))
echo -n "Run-Tests raspberrypi remote..."
echo "$PREFIX_OUT_LABEL" "Run-Tests raspberrypi remote..." >>$TEST_OUT_FILE 2>&1
if [ $ARM_TEST_MACHINE_AVAIL ]; then
	STAGE_STARTAT_INT=$(date +%s)
	for i in $RASPBERRYPICONFIGS; do 
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN >>$TEST_OUT_FILE 2>&1
		NUM_PASSES_OF_REGTESTS_RUN=$(($NUM_PASSES_OF_REGTESTS_RUN + 1))
	done; 
	for i in $RASPBERRYPIVALGRINDCONFIGS; do 
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=memcheck" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=memcheck >>$TEST_OUT_FILE 2>&1
		echo "$PREFIX_OUT_LABEL" "make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=helgrind" >>$TEST_OUT_FILE 2>&1
		make run-tests CONFIGURATION=$i REMOTE=$RASPBERRYPI_REMOTE_WITH_LOGIN VALGRIND=helgrind >>$TEST_OUT_FILE 2>&1
{'fraction_non_alphanumeric': 0.15249926057379473, 'fraction_numerical': 0.01892931085477669, 'mean_word_length': 5.455135547919053, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 270, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '492729', 'n_tokens_mistral': 7176, 'n_tokens_neox': 6614, 'n_words': 1308}