<commit_before>#!/bin/sh

# JENKINS_VERBOSE 
#     TRUE : print logs and stdio while running the tests
#     else : print logs and stdio only on failure, after running the test
#
# JENKINS_EXIT_ON_FAIL:
#     TRUE : exit immediately when a test fails
#     else : run all tests, independent of success/failure (exit code reflects
#            failures though)

# make sure we exit cleanly
\trap shutdown QUIT TERM EXIT

export FAILED=0

export TEST_OK="JENKINS TEST SUCCESS"

export SAGA_VERBOSE=DEBUG
export RADICAL_VERBOSE=DEBUG
export RADICAL_UTILS_VERBOSE=DEBUG
export RADICAL_PILOT_VERBOSE=DEBUG

export TGT="../report"

export HTML_TARGET="$TGT/test_results.html"
export HTML_SUCCESS="<font color=\"\#66AA66\">SUCCESS</font>"
export HTML_FAILURE="<font color=\"\#AA6666\">FAILED</font>"

# ------------------------------------------------------------------------------
#
html_start()
{
    (
        echo "<html>"
        echo " <body>"
        echo "  <table>"
        echo "   <tr>"
        echo "    <td> <b> Test    </b> </td> "
        echo "    <td> <b> Result  </b> </td> "
        echo "    <td> <b> Logfile </b> </td> "
        echo "    <td> <b> Stats   </b> </td> "
        echo "    <td> <b> Plot    </b> </td> "
        echo "  </tr>"
    ) > $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
html_entry()
{
    name=$1
    result=$2
    logfile=$3
    sid=$4

    (
        echo "  <tr>"
        echo "   <td> $name    </td> "
        echo "   <td> $result  </td> "
        echo "   <td> <a href=\"$logfile\">log</a> </td> "
        if test -f "$TGT/$sid.txt"
        then
            echo "   <td> <a href=\"$sid.txt\">stat</a> </td> "
        else
            echo "   <td> - </td> "
        fi
        if test -f "$TGT/$sid.png"
        then
            echo "   <td> <a href=\"$sid.png\">plot</a> </td> "
        else
            echo "   <td> - </td> "
        fi
        echo " </tr>"
    ) >> $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
html_stop()
{
    (
        echo "  </table>"
        echo " </body>"
        echo "</html>"
    ) >> $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
run_test() {

    name="$1";  shift
    cmd="$*"

    echo "# -----------------------------------------------------"
    echo "# TEST $name: $cmd"
    echo "# "

    log="$TGT/$name.log"

    if ! test -z "$JENKINS_VERBOSE"
    then
        progress='print'
    else
        progress='printf "."'
    fi

    (set -e ; $cmd ; printf "\n$TEST_OK\n") 2>&1 | tee "$log" | awk "1==NR%80{print \"\"}{$progress}"
    echo

    SID=`grep 'SESSION ID' $log | head -n 1 | cut -f 2 -d ':' | tr -d ' '`
    if ! test -z "$SID"
    then
        radicalpilot-stats         -m  stat,plot -s $SID -t png >  $TGT/$SID.txt 2>&1
        radicalpilot-close-session -m  purge     -s $SID        >> $TGT/$SID.txt 2>&1
        test -f $SID.png && mv $SID.png $TGT/
    fi


    if grep -q "$TEST_OK" "$log"
    then
        html_entry "$name" "$HTML_SUCCESS" "$log" $SID
        echo "# "
        echo "# SUCCESS $s $t"
        echo "# -----------------------------------------------------"
    else
        html_entry "$name" "$HTML_FAILURE" "$log" $SID
        echo "# "
        echo "# FAILED $s $t"
        echo "# -----------------------------------------------------"

        FAILED=1
    fi


    if test "$JENKINS_EXIT_ON_FAIL" = "TRUE" && test "$FAILED" -eq 0
    then
        shutdown
    fi
}

<commit_msg>iteration on test logging<commit_after>#!/bin/sh

# JENKINS_VERBOSE 
#     TRUE : print logs and stdio while running the tests
#     else : print logs and stdio only on failure, after running the test
#
# JENKINS_EXIT_ON_FAIL:
#     TRUE : exit immediately when a test fails
#     else : run all tests, independent of success/failure (exit code reflects
#            failures though)

# make sure we exit cleanly
\trap shutdown QUIT TERM EXIT

export FAILED=0

export TEST_OK="JENKINS TEST SUCCESS"

export SAGA_VERBOSE=DEBUG
export RADICAL_VERBOSE=DEBUG
export RADICAL_UTILS_VERBOSE=DEBUG
export RADICAL_PILOT_VERBOSE=DEBUG

export FILE_TGT="../report"
export LINK_TGT="http://ci.radical-project.org/job/radical.pilot.devel/branch/devel/ws/radical.pilot/report/"

export HTML_TARGET="$FILE_TGT/test_results.html"
export HTML_SUCCESS="<font color=\"\#66AA66\">SUCCESS</font>"
export HTML_FAILURE="<font color=\"\#AA6666\">FAILED</font>"

# ------------------------------------------------------------------------------
#
html_start()
{
    (
        echo "<html>"
        echo " <body>"
        echo "  <table>"
        echo "   <tr>"
        echo "    <td> <b> Test    </b> </td> "
        echo "    <td> <b> Result  </b> </td> "
        echo "    <td> <b> Logfile </b> </td> "
        echo "    <td> <b> Stats   </b> </td> "
        echo "    <td> <b> Plot    </b> </td> "
        echo "  </tr>"
    ) > $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
html_entry()
{
    name=$1
    result=$2

    (
        echo "  <tr>"
        echo "   <td> $name    </td> "
        echo "   <td> $result  </td> "
        if test -f "$FILE_TGT/$name.log"
        then
            echo "   <td> <a href=\"$LINK_TGT/$name.log\">stat</a> </td> "
        else
            echo "   <td> - </td> "
        fi
        if test -f "$FILE_TGT/$name.txt"
        then
            echo "   <td> <a href=\"$LINK_TGT/$name.txt\">stat</a> </td> "
        else
            echo "   <td> - </td> "
        fi
        if test -f "$FILE_TGT/$name.png"
        then
            echo "   <td> <a href=\"$LINK_TGT/$name.png\">plot</a> </td> "
        else
            echo "   <td> - </td> "
        fi
        echo " </tr>"
    ) >> $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
html_stop()
{
    (
        echo "  </table>"
        echo " </body>"
        echo "</html>"
    ) >> $HTML_TARGET
}


# ------------------------------------------------------------------------------
#
run_test() {

    name="$1";  shift
    cmd="$*"

    echo "# -----------------------------------------------------"
    echo "# TEST $name: $cmd"
    echo "# "

    log="$FILE_TGT/$name.log"

    if ! test -z "$JENKINS_VERBOSE"
    then
        progress='print'
    else
        progress='printf "."'
    fi

    (set -e ; $cmd ; printf "\n$TEST_OK\n") 2>&1 | tee "$log" | awk "1==NR%80{print \"\"}{$progress}"
    echo

    SID=`grep 'SESSION ID' $log | head -n 1 | cut -f 2 -d ':' | tr -d ' '`
    if ! test -z "$SID"
    then
        radicalpilot-stats         -m  stat,plot -s $SID -t png >  $FILE_TGT/$name.txt 2>&1
        radicalpilot-close-session -m  purge     -s $SID        >> $FILE_TGT/$name.txt 2>&1
        test -f $SID.png && mv $SID.png $FILE_TGT/$name.png
    fi


    if grep -q "$TEST_OK" "$log"
    then
        html_entry "$name" "$HTML_SUCCESS"
        echo "# "
        echo "# SUCCESS $s $t"
        echo "# -----------------------------------------------------"
    else
        html_entry "$name" "$HTML_FAILURE"
        echo "# "
        echo "# FAILED $s $t"
        echo "# -----------------------------------------------------"

        FAILED=1
    fi


    if test "$JENKINS_EXIT_ON_FAIL" = "TRUE" && test "$FAILED" -eq 0
    then
        shutdown
    fi
}


{'fraction_non_alphanumeric': 0.27303385943612574, 'fraction_numerical': 0.00701470389855659, 'mean_word_length': 1.261055199756023, 'pattern_counts': {'":': 0, '<': 113, '<?xml version=': 0, '>': 135, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1152848', 'n_tokens_mistral': 2478, 'n_tokens_neox': 2398, 'n_words': 599}