<commit_before>#!/bin/bash
set -eo pipefail
# variables
echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment"
GIT_ROOT="$(dirname $BASH_SOURCE[0])/.."
[[ -z "$TEST" ]] && export TEST=$1
if [[ "$(uname)" == 'Linux' ]]; then
    . /etc/os-release
    if [[ "$ID" == 'centos' ]]; then
        [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable
    fi
    cd $GIT_ROOT && npm install
else
    npm install
fi
cd $GIT_ROOT/build
# tests
if [[ -z "$TEST" ]]; then # run all serial tests
    # count tests
    echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Non-Parallelizable Tests"
    TEST_COUNT=$(ctest -N -L nonparallelizable_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}')
    if [[ $TEST_COUNT > 0 ]]; then
        echo "$TEST_COUNT tests found."
        # run tests
        set +e # defer ctest error handling to end
        CTEST_COMMAND="ctest -L 'nonparallelizable_tests' --output-on-failure -T 'Test'"
        echo "$ $CTEST_COMMAND"
        eval $CTEST_COMMAND
        EXIT_STATUS=$?
        echo 'Done running non-parallelizable tests.'
    else
        echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..."
        EXIT_STATUS='1'
    fi
else # run specific serial test
    # ensure test exists
    echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST"
    TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}')
    if [[ $TEST_COUNT > 0 ]]; then
        echo "$TEST found."
        # run tests
        set +e # defer ctest error handling to end
        CTEST_COMMAND="ctest -R '^$TEST$' -V -T 'Test' > ctest-output.log 2>&1"
        echo "$ $CTEST_COMMAND"
        eval $CTEST_COMMAND
        EXIT_STATUS=$?
        if  [[ $EXIT_STATUS -ne 0 ]]; then
            cat ctest-output.log
            echo "$TEST FAILED - see ctest-output.log in archive"
        else
            echo "$TEST PASSED"
        fi
        grep "produced the following blocks late" ctest-output.log
        echo "Done running $TEST."
    else
        echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests matching \"$TEST\" registered with ctest! Exiting..."
        EXIT_STATUS='1'
    fi
fi
exit $EXIT_STATUS<commit_msg>Print test logs in realtime; organize following steps<commit_after>#!/bin/bash
set -eo pipefail
# variables
echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment"
GIT_ROOT="$(dirname $BASH_SOURCE[0])/.."
[[ -z "$TEST" ]] && export TEST=$1
if [[ "$(uname)" == 'Linux' ]]; then
    . /etc/os-release
    if [[ "$ID" == 'centos' ]]; then
        [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable
    fi
    cd $GIT_ROOT && npm install
else
    npm install
fi
cd $GIT_ROOT/build
# tests
if [[ -z "$TEST" ]]; then # run all serial tests
    # count tests
    echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Non-Parallelizable Tests"
    TEST_COUNT=$(ctest -N -L nonparallelizable_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}')
    if [[ $TEST_COUNT > 0 ]]; then
        echo "$TEST_COUNT tests found."
        # run tests
        set +e # defer ctest error handling to end
        CTEST_COMMAND="ctest -L 'nonparallelizable_tests' --output-on-failure -T 'Test'"
        echo "$ $CTEST_COMMAND"
        eval $CTEST_COMMAND
        EXIT_STATUS=$?
        echo 'Done running non-parallelizable tests.'
    else
        echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..."
        EXIT_STATUS='1'
    fi
else # run specific serial test
    # ensure test exists
    echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST"
    TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}')
    if [[ $TEST_COUNT > 0 ]]; then
        echo "$TEST found."
        # run tests
        set +e # defer ctest error handling to end
        CTEST_COMMAND="ctest -R '^$TEST$' -V -T 'Test' 2>&1 | tee 'ctest-output.log'"
        echo "$ $CTEST_COMMAND"
        eval $CTEST_COMMAND
        EXIT_STATUS=$?
        echo "Done running $TEST."
        [[ "$EXIT_STATUS" == '0' ]] && echo "$TEST PASSED" || echo "$TEST FAILED - see ctest-output.log in the artifacts tab"
        echo 'Late blocks:'
        LATE_BLOCKS="grep -n 'produced the following blocks late' 'ctest-output.log'"
        echo "$ $LATE_BLOCKS"
        eval $LATE_BLOCKS
    else
        echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests matching \"$TEST\" registered with ctest! Exiting..."
        EXIT_STATUS='1'
    fi
fi
exit $EXIT_STATUS
{'fraction_non_alphanumeric': 0.1719692243709711, 'fraction_numerical': 0.007070076939072572, 'mean_word_length': 2.581533879374535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422506', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1612, 'n_words': 480}