<commit_before>#!/usr/bin/env bash

set -xe

NEUTRON_DIR="$BASE/new/neutron"
SCRIPTS_DIR="/usr/os-testr-env/bin/"

venv=${1:-"dsvm-functional"}

function generate_testr_results {
    # Give job user rights to access tox logs
    sudo -H -u $owner chmod o+rw .
    sudo -H -u $owner chmod o+rw -R .testrepository
    if [ -f ".testrepository/0" ] ; then
        .tox/$venv/bin/subunit-1to2 < .testrepository/0 > ./testrepository.subunit
        $SCRIPTS_DIR/subunit2html ./testrepository.subunit testr_results.html
        gzip -9 ./testrepository.subunit
        gzip -9 ./testr_results.html
        sudo mv ./*.gz /opt/stack/logs/
    fi
}

if [[ "$venv" == dsvm-functional* ]] || [[ "$venv" == dsvm-fullstack* ]]; then
    owner=stack
    sudo_env=

    # Set owner permissions according to job's requirements.
    cd $NEUTRON_DIR
    sudo chown -R $owner:stack $NEUTRON_DIR

    # Run tests
    echo "Running neutron $venv test suite"
    set +e
    sudo -H -u $owner $sudo_env tox -e $venv
    testr_exit_code=$?
    set -e

    # Collect and parse results
    generate_testr_results
    exit $testr_exit_code
fi<commit_msg>Generate index file containing oslo.log messages for all tests<commit_after>#!/usr/bin/env bash

set -xe

NEUTRON_DIR="$BASE/new/neutron"
SCRIPTS_DIR="/usr/os-testr-env/bin/"

venv=${1:-"dsvm-functional"}

function generate_testr_results {
    # Give job user rights to access tox logs
    sudo -H -u $owner chmod o+rw .
    sudo -H -u $owner chmod o+rw -R .testrepository
    if [ -f ".testrepository/0" ] ; then
        .tox/$venv/bin/subunit-1to2 < .testrepository/0 > ./testrepository.subunit
        $SCRIPTS_DIR/subunit2html ./testrepository.subunit testr_results.html
        gzip -9 ./testrepository.subunit
        gzip -9 ./testr_results.html
        sudo mv ./*.gz /opt/stack/logs/
    fi
}

function generate_log_index {
    local xtrace
    xtrace=$(set +o | grep xtrace)
    set +o xtrace

    virtualenv /tmp/os-log-merger
    /tmp/os-log-merger/bin/pip install -U os-log-merger==1.0.6
    files=$(find /opt/stack/logs/$venv-logs -name '*.txt' -o -name '*.log')
    # -a3 to truncate common path prefix
    contents=$(/tmp/os-log-merger/bin/os-log-merger -a3 $files)
    # don't store DEBUG level messages because they are not very useful,
    # and are not indexed by logstash anyway
    echo "$contents" | grep -v DEBUG | sudo tee /opt/stack/logs/$venv-index.txt > /dev/null

    $xtrace
}

if [[ "$venv" == dsvm-functional* ]] || [[ "$venv" == dsvm-fullstack* ]]; then
    owner=stack
    sudo_env=

    # Set owner permissions according to job's requirements.
    cd $NEUTRON_DIR
    sudo chown -R $owner:stack $NEUTRON_DIR

    # Run tests
    echo "Running neutron $venv test suite"
    set +e
    sudo -H -u $owner $sudo_env tox -e $venv
    testr_exit_code=$?
    set -e

    # Collect and parse results
    generate_testr_results
    generate_log_index
    exit $testr_exit_code
fi
{'fraction_non_alphanumeric': 0.12580975110808046, 'fraction_numerical': 0.007159904534606206, 'mean_word_length': 2.695214105793451, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5202916', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1045, 'n_words': 275}