<commit_before>#!/bin/sh
#
# initialize the testdata directory on the installed VM 
#
#
#======================================================================

. `dirname $0`/../lib_logging.sh

set -x 
TD_DIR="/srv/eodata/testdata"
DOWNLOAD_SH="$TD_DIR/download_test_data.sh"
info "Initializing the test data directory $TD_DIR"

[ -z "$ODAOSUSER" ] && error "Missing the required ODAOSUSER variable!"
[ -z "$ODAOSGROUP" ] && error "Missing the required ODAOSGROUP variable!"

if [ -f "$TD_DIR" ]
then 
    rm -fR "$TD_DIR"
fi

mkdir -p "$TD_DIR"
chown -R "$ODAOSUSER:$ODAOSGROUP" "$TD_DIR"

cat > "$DOWNLOAD_SH" <<END
#!/bin/sh -e 
#
# automatic data download script
#
cd "\`dirname "\$0"\`"
URL_BASE="http://dream.eox.at/testdata"
if [ -n "\$USER" -a -n "\$PASSWD" ] 
then
    OPT="--no-host-directories --no-parent --cut-dirs=1 -nv"
    for C in ENVISAT SPOT4Take5 GISAT
    do
        echo "Collection: $C"
        [ -d "\$C" -o -f "\$C" ] && rm -fvR "\$C" || true
        wget --user "\$USER" --password "\$<PASSWORD>" \$OPT -r "\$URL_BASE/\$C/"
        [ ! -d "\$C" ] || find "\$C" -name 'index.html*' -exec rm -fv {} \\;
    done 
    echo "DOWNLOAD FINISHED"
else 
    echo "Startig the automated test data download script ..."
    read -p "Enter username: " USER
    read -s -p "Enter password: " <PASSWORD>WD
    export USER 
    export PASSWD
    echo "Startig background background download process ... "
    echo "To see the progress run: tail -f \$0.log"
    nohup \$0 >\$0.log 2>&1 &
    echo "To stop the process run: kill \$!"
fi 

END
chown -R "$ODAOSUSER:$ODAOSGROUP" "$DOWNLOAD_SH"
chmod +x "$DOWNLOAD_SH"<commit_msg>Update of the testdata download.<commit_after>#!/bin/sh
#
# initialize the testdata directory on the installed VM 
#
#
#======================================================================

. `dirname $0`/../lib_logging.sh

set -x 
TD_DIR="/srv/eodata/testdata"
DOWNLOAD_SH="$TD_DIR/download_test_data.sh"
info "Initializing the test data directory $TD_DIR"

[ -z "$ODAOSUSER" ] && error "Missing the required ODAOSUSER variable!"
[ -z "$ODAOSGROUP" ] && error "Missing the required ODAOSGROUP variable!"

if [ -f "$TD_DIR" ]
then 
    rm -fR "$TD_DIR"
fi

mkdir -p "$TD_DIR"
chown -R "$ODAOSUSER:$ODAOSGROUP" "$TD_DIR"

cat > "$DOWNLOAD_SH" <<END
#!/bin/sh -e 
#
# automatic data download script
#
cd "\`dirname "\$0"\`"
URL_BASE="http://dream.eox.at/testdata"
if [ -n "\$USER" -a -n "\$PASSWD" ] 
then
    OPT="--no-host-directories --no-parent --cut-dirs=1 -nv"
    for C in ENVISAT SPOT4Take5 GISAT
    do
        echo "Collection: $C"
        [ -d "\$C" -o -f "\$C" ] && rm -fvR "\$C" || true
        wget --user "\$USER" --password "\$<PASSWORD>" \$OPT -r "\$URL_BASE/\$C/"
        [ ! -d "\$C" ] || find "\$C" -name 'index.html*' -exec rm -fv {} \\;
    done 
    echo "DOWNLOAD FINISHED"
else 
    echo "Startig the automated test data download script ..."
    read -p "Enter username: " USER
    read -s -p "Enter password: " PASSWD
    export USER 
    export PASSWD
    echo "Startig background background download process ... "
    echo "To see the progress run: tail -f \$0.log"
    nohup \$0 >\$0.log 2>&1 &
    PID="\$!"
    echo "The background download may take hours to finish."
    echo "It is safe to logout and check the progress later."
    echo "The PID of the background process is: \$PID"
    echo "To kill the process use: kill -9 \$PID ; killall wget"
fi 

END
chown -R "$ODAOSUSER:$ODAOSGROUP" "$DOWNLOAD_SH"
chmod +x "$DOWNLOAD_SH"
{'fraction_non_alphanumeric': 0.2037725064304087, 'fraction_numerical': 0.006001714775650186, 'mean_word_length': 2.9548022598870056, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1994210', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1258, 'n_words': 389}