<commit_before>#!/bin/bash

SRCDIR=$( cd -P $(dirname ${BASH_SOURCE[0]}); echo $PWD )

source $SRCDIR/checkVirtualEnv.sh
source $SRCDIR/spark_config.sh

time spark-submit  \
--verbose \
--master local[*] \
--driver-memory 13G \
--packages com.databricks:spark-csv_2.10:1.4.0 \
--conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" \
--conf "spark.eventLog.enabled=true" \
--conf "spark.eventLog.dir=$EVENT_LOG_DIR" \
--conf "spark.local.dir=$LOCALDIR" \
$SRCDIR/TrainModel.py > $SRCDIR/runml.out 2> $SRCDIR/runml.err
RETVAL=$?

echo "status $RETVAL"
exit $RETVAL<commit_msg>add datestamp to runml output<commit_after>#!/bin/bash

SRCDIR=$( cd -P $(dirname ${BASH_SOURCE[0]}); echo $PWD )

source $SRCDIR/checkVirtualEnv.sh
source $SRCDIR/spark_config.sh

NOW=$(date +"%Y%m%d%H%M%S")

time spark-submit  \
--verbose \
--master local[*] \
--driver-memory 13G \
--packages com.databricks:spark-csv_2.10:1.4.0 \
--conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" \
--conf "spark.eventLog.enabled=true" \
--conf "spark.eventLog.dir=$EVENT_LOG_DIR" \
--conf "spark.local.dir=$LOCALDIR" \
$SRCDIR/TrainModel.py > $SRCDIR/runml.${NOW}.out 2> $SRCDIR/runml.${NOW}.err
RETVAL=$?

echo "status $RETVAL"
exit $RETVAL
{'fraction_non_alphanumeric': 0.19179163378058406, 'fraction_numerical': 0.01578531965272297, 'mean_word_length': 4.51304347826087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751756', 'n_tokens_mistral': 537, 'n_tokens_neox': 512, 'n_words': 85}