<commit_before>#!/bin/bash
#BUILD_DIR=${BUILD_DIR:=$HOME/raven_libs/build}
#INSTALL_DIR=${INSTALL_DIR:=$HOME/raven_libs/pylibs}
PYTHON_CMD=${PYTHON_CMD:=python}
JOBS=${JOBS:=1}
SCRIPT_DIRNAME=`dirname $0`
SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)`

ORIGPYTHONPATH="$PYTHONPATH"

maxlen=$(($(tput cols) < 100 ? $(tput cols) : 100))

TEST_DIRS="${SCRIPT_DIR}/../tests/cluster_tests "`find ${SCRIPT_DIR}/../tests/framework -name tests -printf "%h\n"`
echo $TEST_DIRS
failed_tests=0
passed_tests=0

CONVERSION_SCRIPT_DIR="${SCRIPT_DIR}/../scripts/conversionScripts"

for testdir in ${TEST_DIRS}
do
    echo -e "\033[1;32mValidating $testdir "
    cd $testdir
    for I in $(python ${SCRIPT_DIR}/get_coverage_tests.py)
    do
        # convert the input files contain the external xml to normal raven input files and then validate
        $PYTHON_CMD $CONVERSION_SCRIPT_DIR/externalXMLNode.py $I > /dev/null
        # echo -e "\033[1;32mThe following script: $I has been converted for xsd validate"
        echo -en "\033[0mValidating $I"
        startlen=$((11+${#I}))
        VALOUT=$(xmllint --noout --schema  ${SCRIPT_DIR}/XSDSchemas/raven.xsd $I 2>&1)
        if test $? -eq 0;
        then
            periodlength=$((maxlen - startlen - 11))
            printf '%0.s.' $(seq 1 $periodlength)
            echo -e "\033[1;32m validated!"
            passed_tests=$(($passed_tests + 1))
        else
            periodlength=$((maxlen - startlen - 8))
            printf '%0.s.' $(seq 1 $periodlength)
            echo -e "\033[1;31m FAILED!"
            echo -e "$VALOUT"
            failed_tests=$(($failed_tests + 1))
        fi
        # Move the file back to its original state
        mv $I.bak $I
        # echo -e "\033[1;32m$I has been converted back!"
    done
    echo -e "\033[0m--------------------------------------------------"
done
echo -e "\033[1;32mPassed $passed_tests \033[1;31mFailed $failed_tests\033[0m"

# test_External_XML.xml should transformed back, this is the test that test the external xml input functionality
# mv ${SCRIPT_DIR}/../tests/framework/test_External_XML.xml.bak ${SCRIPT_DIR}/../tests/framework/test_External_XML.xml
# echo -e "\033[1;32m test_External_XML.xml has been converted back!"

exit $failed_tests
<commit_msg>Creating a Mac-compatible way of finding the appropriate test directories.<commit_after>#!/bin/bash
#BUILD_DIR=${BUILD_DIR:=$HOME/raven_libs/build}
#INSTALL_DIR=${INSTALL_DIR:=$HOME/raven_libs/pylibs}
PYTHON_CMD=${PYTHON_CMD:=python}
JOBS=${JOBS:=1}
SCRIPT_DIRNAME=`dirname $0`
SCRIPT_DIR=`(cd $SCRIPT_DIRNAME; pwd)`

ORIGPYTHONPATH="$PYTHONPATH"

maxlen=$(($(tput cols) < 100 ? $(tput cols) : 100))

# Not Mac-compatible:
#TEST_DIRS="${SCRIPT_DIR}/../tests/cluster_tests "`find ${SCRIPT_DIR}/../tests/framework -name tests -printf "%h\n"`
TEST_FILES="${SCRIPT_DIR}/../tests/cluster_tests "`find ${SCRIPT_DIR}/../tests/framework -name tests`
TEST_DIRS=""
for file in ${TEST_FILES}
do
    TEST_DIRS+=" $(dirname $file)"
done

failed_tests=0
passed_tests=0

CONVERSION_SCRIPT_DIR="${SCRIPT_DIR}/../scripts/conversionScripts"

for testdir in ${TEST_DIRS}
do
    echo -e "\033[1;32mValidating $testdir "
    cd $testdir
    for I in $(python ${SCRIPT_DIR}/get_coverage_tests.py)
    do
        # convert the input files contain the external xml to normal raven input files and then validate
        $PYTHON_CMD $CONVERSION_SCRIPT_DIR/externalXMLNode.py $I > /dev/null
        # echo -e "\033[1;32mThe following script: $I has been converted for xsd validate"
        echo -en "\033[0mValidating $I"
        startlen=$((11+${#I}))
        VALOUT=$(xmllint --noout --schema  ${SCRIPT_DIR}/XSDSchemas/raven.xsd $I 2>&1)
        if test $? -eq 0;
        then
            periodlength=$((maxlen - startlen - 11))
            printf '%0.s.' $(seq 1 $periodlength)
            echo -e "\033[1;32m validated!"
            passed_tests=$(($passed_tests + 1))
        else
            periodlength=$((maxlen - startlen - 8))
            printf '%0.s.' $(seq 1 $periodlength)
            echo -e "\033[1;31m FAILED!"
            echo -e "$VALOUT"
            failed_tests=$(($failed_tests + 1))
        fi
        # Move the file back to its original state
        mv $I.bak $I
        # echo -e "\033[1;32m$I has been converted back!"
    done
    echo -e "\033[0m--------------------------------------------------"
done
echo -e "\033[1;32mPassed $passed_tests \033[1;31mFailed $failed_tests\033[0m"

# test_External_XML.xml should transformed back, this is the test that test the external xml input functionality
# mv ${SCRIPT_DIR}/../tests/framework/test_External_XML.xml.bak ${SCRIPT_DIR}/../tests/framework/test_External_XML.xml
# echo -e "\033[1;32m test_External_XML.xml has been converted back!"

exit $failed_tests

{'fraction_non_alphanumeric': 0.16341514579400043, 'fraction_numerical': 0.03524229074889868, 'mean_word_length': 2.943755169561621, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4868520', 'n_tokens_mistral': 1769, 'n_tokens_neox': 1605, 'n_words': 392}