<commit_before>#!/bin/bash

export PATH=/bin:/usr/bin
cur_dir=`pwd`
# save the first three arguments and then discard
source=$1
es_index=$2
processed=$3
shift
shift
shift
while [ "$1" != "" ]; do
    if [ "$processed" == 'True' ];
    then
        ./download_logs.py --date $1 --source ${source} --processed
    else
        ./download_logs.py --date $1 --source ${source}
        ./process_logs.py --date $1

    fi
    shift
done

sed -i "s/ES_INDEX/$es_index/" joblog.conf
cat *-processed.csv | /opt/logstash/bin/logstash -f joblog.conf
rm *.csv<commit_msg>Give the cluster a little time to catch up after going through each log file<commit_after>#!/bin/bash

export PATH=/bin:/usr/bin
cur_dir=`pwd`
# save the first three arguments and then discard
source=$1
es_index=$2
processed=$3
sed -i "s/ES_INDEX/$es_index/" joblog.conf
shift
shift
shift
while [ "$1" != "" ]; do
    if [ "$processed" == 'True' ];
    then
        ./download_logs.py --date $1 --source ${source} --processed
        cat jobsarchived$1-processed.csv | /opt/logstash/bin/logstash -f joblog.conf
        sleep 120
    else
        ./download_logs.py --date $1 --source ${source}
        ./process_logs.py --date $1
        cat jobsarchived$1-processed.csv | /opt/logstash/bin/logstash -f joblog.conf
        sleep 120
    fi
    shift
done

rm *.csv
{'fraction_non_alphanumeric': 0.14652567975830816, 'fraction_numerical': 0.01661631419939577, 'mean_word_length': 2.660220994475138, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3934349', 'n_tokens_mistral': 482, 'n_tokens_neox': 448, 'n_words': 130}