<commit_before># PARAMS_BASE - global environment variable. It is a connection string to database that contains
#               test parameters. If you leave it undefined the test will not get parameters
#               from a database but you define them manually.
test=$1
publish_to=$2
publish_path=$3
publish_login=$4
publish_pass=$5

#Removing cache on the node
sudo rm -rf ~/.cache
sudo pip install --upgrade ndg-httpsclient
sudo pip install -H -r requirements.txt

if [[ -n "${PARAMS_BASE}" ]]; then
    echo "Get parameters..."
    python tools/parameters.py allocate --connection=${PARAMS_BASE} --wait > params.sh
    source params.sh
    echo "*********************** Parameters *********************** "
    cat params.sh
    echo "********************************************************** "
fi

nosetests ${test}
rc=$?

if [[ -n "${PARAMS_BASE}" ]]; then
    echo "Release parameters..."
    python tools/parameters.py release --connection=${PARAMS_BASE} --id=${PARAM_ID}
fi

echo "gzip logs folder"
cd ${WORKSPACE}
find logs -regex '.*log$' -exec gzip -9 {} \;

if [[ -n "${publish_to}" ]]; then
    echo "Publish test results..."
    logspath=${publish_path}/${JOB_NAME}/
    cp -vr logs ${BUILD_NUMBER}
    sshpass -p ${publish_pass} ssh -o StrictHostKeyChecking=no ${publish_login}@${publish_to} mkdir -p ${logspath}
    sshpass -p ${publish_pass} rsync -ave "ssh -o StrictHostKeyChecking=no" ${BUILD_NUMBER} ${publish_login}@${publish_to}:${logspath}
fi

if [[ $rc != 0 ]]
then
    echo "Test FAILED!"
    exit $rc
fi
echo "Test PASSED!"<commit_msg>Fix cash cleanup<commit_after># PARAMS_BASE - global environment variable. It is a connection string to database that contains
#               test parameters. If you leave it undefined the test will not get parameters
#               from a database but you define them manually.
test=$1
publish_to=$2
publish_path=$3
publish_login=$4
publish_pass=$5

#Removing cache on the node
echo "runnin sudo rm -f /home/ubuntu/.cache/pip/http"
sudo rm -f /home/ubuntu/.cache/pip/http
sudo pip install --upgrade ndg-httpsclient
sudo pip install -r requirements.txt

if [[ -n "${PARAMS_BASE}" ]]; then
    echo "Get parameters..."
    python tools/parameters.py allocate --connection=${PARAMS_BASE} --wait > params.sh
    source params.sh
    echo "*********************** Parameters *********************** "
    cat params.sh
    echo "********************************************************** "
fi

nosetests ${test}
rc=$?

if [[ -n "${PARAMS_BASE}" ]]; then
    echo "Release parameters..."
    python tools/parameters.py release --connection=${PARAMS_BASE} --id=${PARAM_ID}
fi

echo "gzip logs folder"
cd ${WORKSPACE}
find logs -regex '.*log$' -exec gzip -9 {} \;

if [[ -n "${publish_to}" ]]; then
    echo "Publish test results..."
    logspath=${publish_path}/${JOB_NAME}/
    cp -vr logs ${BUILD_NUMBER}
    sshpass -p ${publish_pass} ssh -o StrictHostKeyChecking=no ${publish_login}@${publish_to} mkdir -p ${logspath}
    sshpass -p ${publish_pass} rsync -ave "ssh -o StrictHostKeyChecking=no" ${BUILD_NUMBER} ${publish_login}@${publish_to}:${logspath}
fi

if [[ $rc != 0 ]]
then
    echo "Test FAILED!"
    exit $rc
fi
echo "Test PASSED!"
{'fraction_non_alphanumeric': 0.1937597503900156, 'fraction_numerical': 0.00436817472698908, 'mean_word_length': 3.7994011976047903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180431', 'n_tokens_mistral': 1008, 'n_tokens_neox': 978, 'n_words': 320}