<commit_before># 
# This is a script that essentially runs APPIAN a bunch of different times with varying options. In theory, it should
# be useable with any data set, although in practice it has only been developed using the CIMBI data set availble
# on OpenNeuro. 
#
# The purpose of this validatation script is to check that changes made to the APPIAN code do not break the package.
# As such it should be run before pushing any new changes to the Git repository and especially before creating an
# updated Docker container with a new version APPIAN.
#
# More tests will need to be added in the future as the current set are not exhaustive. Each test can take a long time.
# It is therefore a good idea to reuse the output of previous tests to avoid rerunning processing stages unecessarily.
# For example, there is no need to rerun PET-MRI co-registration everytime one wants to test a downstream processing
# stage, like PVC or quantification. The tests are therefore organized such that at least some of the outputs of the 
# previous tests can be reused for subsequent ones. 
#

run_appian() {
    test_name=$1 #Name of the test command to run on APPIAN
    extra_options=${2:-""} #Options beyond the minimal ones necessary to run APPIAN (e.g., pvc, tka, atlases, etc.)
    log=${test_dir}/test_${test_name}.txt  #Name of the log file where stdout/stderr from test will be saved

    if [[ -f ${test_dir}/test_${test_name}.passed ]]; then
        #If log with ".passed" suffix exists, then do not rerun the test
        printf "Test : $test_name -- already passed for timestamp $ts --> skipping\n"
        return 0
    else
        #Test APPIAN with minimal and extra options
        printf "Test: $test_name -- "
        
        #Define variable with minimal command to launch APPIAN
        minimal_options="python2.7 $base_path/Launcher.py -s ${test_data_path} -t ${out_data_path} --threads ${threads}"
        bash -c "$minimal_options $extra_options" &> $log
       
        #The number of errors produced by the APPIAN run is determined by the number of crash reports with
        # .pklz suffix produced by Nipype in the current directory. This is why any existing crash reports must 
        # moved elsewhere before initiating the run
        n_errors=`ls crash-*pklz 2> /dev/null | wc -l`
        printf "Errors = %s -->" "$n_errors"

        if [[ $n_errors != 0 ]]; then
            #If errors are detected, then the log suffix is set to failed
            printf " failed.\n"
            out_log=${test_dir}/test_${test_name}.failed
            crash_files=`ls crash-*pklz &> /dev/null`
            mv $crash_files  $test_dir #Move Nipype crash reports to test directory
        else
            #If errors are detected, then the log suffix is set to passed
            printf " passed.\n"
            out_log=${test_dir}/test_${test_name}.passed
            rm -f ${test_dir}/test_${test_name}.failed #If there is a previous failed log, remove it
        fi
        mv $log $out_log #Move log to version with passed/failed suffix
    fi
}

if [[ $1 == "-help" || $1 == "--help" || $1 == "-h" || $1 == "--h" ]]; then
    echo
    echo 'Name :        appian_validation.sh'
    echo 'Description:  Run validation test suite on current version of APPIAN'
    echo 'Useage :      test.sh <n threads> <path to APPIAN dir> <path to data> <path for outputs> <timestamp>'
    echo "              n threads :             number of CPU threads to use for testing (default=1, recommended=4)"
    echo "              path to APPIAN dir :    Abs. path to location of APPIAN repository (default=/APPIAN)"
    echo "              path to data :          Path to data to be used to testing (default=/APPIAN/Test/cimbi)"
    echo "              path for outputs :      Path where testing outputs will be saved (default=/APPIAN/Test)"
    echo "              timestamp :             Timestamp for validation. Will be set each time scipt is run. "
    echo "                                      However, users that are debugging may wish to continue validation with"
    echo "                                      existing timestamp to prevent rerunning all test. In this case, "
    echo "                                      users can provide timestamp from exisint validation." 
    echo "Examples :"
    echo "          1) Run validation defaults"
    echo "             ./validate_appian.sh"
    echo "          2) Run validation with 8 threads in non-default directories. Timestamp genereated at runtime"
    echo "              ./validate_appian.sh 8 /home/APPIAN /data1/projects/cimbi /data1/projects/  "
    echo "          3) Run with 1 thread in default directories with pre-existing timestamp"
    echo "              ./validate_appian.sh 1 /APPIAN /APPIAN/Test/cimbi /APPIAN/Test 20181221-220648"
    echo
    exit 1
fi
echo
echo "     Launching APPIAN validation tests "
echo "   -------------------------------------"
echo Warning: You should have commited latest changes to current git branch

########################
# Setup input vaiables #
########################

#Number of threads to use for running APPIAN. Default=1, recommended is 4 or more.
threads=${1:-1} 
base_path=${2:-"/APPIAN"}
test_data_path=${3:-"/APPIAN/Test/test_data"}
out_data_path=${4:-"/APPIAN/Test"}
#Create a timestamp for testing session. 
#Can use existing timestamp if using output files from previous test run
ts=${5:-`date +"%Y%m%d-%H%M%S"`}
test_dir="appian_validation/test_$ts"

################
# Check Inputs #
################
if [[ ! -f $base_path/Launcher.py ]]; then
    echo Error: $base_path/Launcher.py does not exist. Must specify valid path to APPIAN repository.
    exit 1
fi

if [[ ! -d $test_data_path ]]; then
    echo Error: $test_data_path does not exist. Must specify valid path to test data directory.
    exit 1
fi

if [[ ! -d $out_data_path ]]; then
    echo Error: $out_data_path does not exist. Must specify valid path where output data can be stored.
    exit 1
fi

if [[ `ls crash*pklz 2> /dev/null` ]]; then
    echo Warning: Moving existin crash .pklz reports to `pwd`/crash_backup
    echo Test script requires that there be no crash reports in the current directory
    mkdir -p nipype_crash_backup
    mv `ls crash*pklz` nipype_crash_backup
fi

#################################
# Create preliminary files/dirs #
#################################

#Create directory to store test results
mkdir -p appian_validation $test_dir

#Get hash for current git commit
current_git_commit=`git rev-parse HEAD`

#Create a .json with some info about the run. Should help keep track of what version of APPIAN was tested
echo "{ "base_path":$base_path, "test_data_path":$test_data_path,"out_data_path":$out_data_path,"git_hash":$current_git_commit,"timestamp":$ts}" > ${test_dir}/info.json

#############
# Run tests #
#############
echo Timestamp: $ts
echo Git Commit : $current_git_commit
echo

### Minimal Inputs
run_appian "Mininimum"

### PVC
pvc_methods="GTM idSURF"
for pvc in $pvc_methods; do
    run_appian "PVC-${pvc}" "--fwhm 3 3 3 --pvc-method ${pvc}" 
done

### Quantification
quant_methods="lp pp suvr" #suv srtm, roi

for quant in $quant_methods; do
    run_appian "Quant-${quant}" "--tka-method ${quant} --tka-label 3 --tka-label-erosion 1"
done

### Atlas / Templates
## DKA atlas in MNI 152 space
run_appian "Atlas-DKA" "--tka-label-img  /APPIAN/Atlas/MNI152/dka.mnc --results-label-img--tka-labels 2,41 --tka-label-ones-only"  
## AAL atlas with Colin 27 template
run_appian "mininimum" "--results-label-img  /APPIAN/Atlas/ROI_MNI_AAL_V5_UBYTE_round.mnc /APPIAN/Atlas/colin27_t1_tal_lin_ubyte.mnc  --threads ${threads}"  
<commit_msg>changed validation so that it considers test to have failed if error code != 0<commit_after># 
# This is a script that essentially runs APPIAN a bunch of different times with varying options. In theory, it should
# be useable with any data set, although in practice it has only been developed using the CIMBI data set availble
# on OpenNeuro. 
#
# The purpose of this validatation script is to check that changes made to the APPIAN code do not break the package.
# As such it should be run before pushing any new changes to the Git repository and especially before creating an
# updated Docker container with a new version APPIAN.
#
# More tests will need to be added in the future as the current set are not exhaustive. Each test can take a long time.
# It is therefore a good idea to reuse the output of previous tests to avoid rerunning processing stages unecessarily.
# For example, there is no need to rerun PET-MRI co-registration everytime one wants to test a downstream processing
# stage, like PVC or quantification. The tests are therefore organized such that at least some of the outputs of the 
# previous tests can be reused for subsequent ones. 
#

run_appian() {
    test_name=$1 #Name of the test command to run on APPIAN
    extra_options=${2:-""} #Options beyond the minimal ones necessary to run APPIAN (e.g., pvc, tka, atlases, etc.)
    log=${test_dir}/test_${test_name}.txt  #Name of the log file where stdout/stderr from test will be saved

    if [[ -f ${test_dir}/test_${test_name}.passed ]]; then
        #If log with ".passed" suffix exists, then do not rerun the test
        printf "Test : $test_name -- already passed for timestamp $ts --> skipping\n"
        return 0
    else
        #Test APPIAN with minimal and extra options
        printf "Test: $test_name -- "
        
        #Define variable with minimal command to launch APPIAN
        minimal_options="python2.7 $base_path/Launcher.py -s ${test_data_path} -t ${out_data_path}/validation_output --threads ${threads}"
        bash -c "$minimal_options $extra_options" &> $log
        errorcode=$? 
        #The number of errors produced by the APPIAN run is determined by the number of crash reports with
        # .pklz suffix produced by Nipype in the current directory. This is why any existing crash reports must 
        # moved elsewhere before initiating the run
        n_errors=`ls crash-*pklz 2> /dev/null | wc -l`
        printf "Errors = %s -->" "$n_errors"
        
        if [[ $n_errors != 0 || $errorcode != 0 ]]; then
            #If errors are detected, then the log suffix is set to failed
            printf " failed.\n"
            out_log=${test_dir}/test_${test_name}.failed
            crash_files=`ls crash-*pklz &> /dev/null`
            if [[ $crash_files ]]; then
                mv $crash_files  ${test_dir}/ #Move Nipype crash reports to test directory
            fi
        else
            #If errors are detected, then the log suffix is set to passed
            printf " passed.\n"
            out_log=${test_dir}/test_${test_name}.passed
            rm -f ${test_dir}/test_${test_name}.failed #If there is a previous failed log, remove it
        fi
        mv $log $out_log #Move log to version with passed/failed suffix
    fi
}

if [[ $1 == "-help" || $1 == "--help" || $1 == "-h" || $1 == "--h" ]]; then
    echo
    echo 'Name :        appian_validation.sh'
    echo 'Description:  Run validation test suite on current version of APPIAN'
    echo 'Useage :      test.sh <n threads> <path to APPIAN dir> <path to data> <path for outputs> <timestamp>'
    echo "              n threads :             number of CPU threads to use for testing (default=1, recommended=4)"
    echo "              path to APPIAN dir :    Abs. path to location of APPIAN repository (default=/APPIAN)"
    echo "              path to data :          Path to data to be used to testing (default=/APPIAN/Test/cimbi)"
    echo "              path for outputs :      Path where testing outputs will be saved (default=/APPIAN/Test)"
    echo "              timestamp :             Timestamp for validation. Will be set each time scipt is run. "
    echo "                                      However, users that are debugging may wish to continue validation with"
    echo "                                      existing timestamp to prevent rerunning all test. In this case, "
    echo "                                      users can provide timestamp from exisint validation." 
    echo "Examples :"
    echo "          1) Run validation defaults"
    echo "             ./validate_appian.sh"
    echo "          2) Run validation with 8 threads in non-default directories. Timestamp genereated at runtime"
    echo "              ./validate_appian.sh 8 /home/APPIAN /data1/projects/cimbi /data1/projects/  "
    echo "          3) Run with 1 thread in default directories with pre-existing timestamp"
    echo "              ./validate_appian.sh 1 /APPIAN /APPIAN/Test/cimbi /APPIAN/Test 20181221-220648"
    echo
    exit 1
fi
echo
echo "     Launching APPIAN validation tests "
echo "   -------------------------------------"
echo Warning: You should have commited latest changes to current git branch

########################
# Setup input vaiables #
########################

#Number of threads to use for running APPIAN. Default=1, recommended is 4 or more.
threads=${1:-1} 
base_path=${2:-"/APPIAN"}
test_data_path=${3:-"/APPIAN/Test/test_data"}
out_data_path=${4:-"/APPIAN/Test/"}
#Create a timestamp for testing session. 
#Can use existing timestamp if using output files from previous test run
ts=${5:-`date +"%Y%m%d-%H%M%S"`}
test_dir="${out_data_path}/appian_validation/test_$ts"

################
# Check Inputs #
################
if [[ ! -f $base_path/Launcher.py ]]; then
    echo Error: $base_path/Launcher.py does not exist. Must specify valid path to APPIAN repository.
    exit 1
fi

if [[ ! -d $test_data_path ]]; then
    echo Error: $test_data_path does not exist. Must specify valid path to test data directory.
    exit 1
fi

if [[ ! -d $out_data_path ]]; then
    echo Error: $out_data_path does not exist. Must specify valid path where output data can be stored.
    exit 1
fi

if [[ `ls crash*pklz 2> /dev/null` ]]; then
    echo Warning: Moving existin crash .pklz reports to `pwd`/crash_backup
    echo Test script requires that there be no crash reports in the current directory
    mkdir -p nipype_crash_backup
    mv `ls crash*pklz` nipype_crash_backup
fi

#################################
# Create preliminary files/dirs #
#################################

#Create directory to store test results
mkdir -p  ${out_data_path}/appian_validation $test_dir

#Get hash for current git commit
current_git_commit=`git rev-parse HEAD`

#Create a .json with some info about the run. Should help keep track of what version of APPIAN was tested
echo "{ "base_path":$base_path, "test_data_path":$test_data_path,"out_data_path":$out_data_path,"git_hash":$current_git_commit,"timestamp":$ts}" > ${test_dir}/info.json

#############
# Run tests #
#############
echo Timestamp: $ts
echo Git Commit : $current_git_commit
echo

### Minimal Inputs
run_appian "Mininimum"

### PVC
pvc_methods="GTM idSURF"
for pvc in $pvc_methods; do
    run_appian "PVC-${pvc}" "--fwhm 6 6 6 --pvc-method ${pvc}" 
done

### Quantification
quant_methods="lp pp suvr" #suv srtm, roi

for quant in $quant_methods; do
    run_appian "Quant-${quant}" "--tka-method ${quant} --tka-label 3 --tka-label-erosion 1"
done

### Analysis Space
## Stereotaxic space
run_appian  "Space-Stereo" "--analysis-space stereo --tka-method suvr --tka-label 3 --tka-label-erosion 1"

## MRI space
run_appian  "Space-MRI" "--analysis-space mri --tka-method suvr --tka-label 3 --tka-label-erosion 1"

### Atlas / Templates
## DKA atlas in MNI 152 space
run_appian "Atlas-DKA" "--tka-label-img  /APPIAN/Atlas/MNI152/dka.mnc --results-label-img--tka-labels 2,41 --tka-label-ones-only"  

## AAL atlas with Colin 27 template
run_appian "Atlas-AAL" "--results-label-img  /APPIAN/Atlas/ROI_MNI_AAL_V5_UBYTE_round.mnc /APPIAN/Atlas/colin27_t1_tal_lin_ubyte.mnc"  

{'fraction_non_alphanumeric': 0.116999747027574, 'fraction_numerical': 0.009107007336200354, 'mean_word_length': 2.859653404930437, 'pattern_counts': {'":': 10, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1841335', 'n_tokens_mistral': 4826, 'n_tokens_neox': 4608, 'n_words': 1804}