<commit_before>#!/usr/bin/env bash

CURRENT_DIR=`pwd`

# spark-lucenerdd assembly JAR
MAIN_JAR=${CURRENT_DIR}/target/scala-2.11/spark-lucenerdd-aws-assembly-0.2.1.jar

# Run spark shell locally
spark-submit   \
	 --driver-memory 2g \
	 --executor-memory 4g \
	 --conf spark.executor.instances=4 \
	 --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
	 --conf spark.kryo.registrator=org.zouzias.spark.lucenerdd.LuceneRDDKryoRegistrator \
	 --conf spark.executor.extraJavaOptions="-Dlucenerdd.index.store.mode=disk" \
	 --conf spark.driver.extraJavaOptions="-Dlucenerdd.index.store.mode=disk" \
	 --master yarn \
	 --deploy-mode client \<commit_msg>read version from version.sbt<commit_after>#!/usr/bin/env bash

CURRENT_DIR=`pwd`

LUCENERDD_VERSION=`cat version.sbt | awk '{print $5}' | xargs`

# spark-lucenerdd assembly JAR
MAIN_JAR=${CURRENT_DIR}/target/scala-2.11/spark-lucenerdd-aws-assembly-${LUCENERDD_VERSION}.jar

# Run spark shell locally
spark-submit   \
	 --driver-memory 2g \
	 --executor-memory 4g \
	 --conf spark.executor.instances=4 \
	 --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
	 --conf spark.kryo.registrator=org.zouzias.spark.lucenerdd.LuceneRDDKryoRegistrator \
	 --conf spark.executor.extraJavaOptions="-Dlucenerdd.index.store.mode=disk" \
	 --conf spark.driver.extraJavaOptions="-Dlucenerdd.index.store.mode=disk" \
	 --master yarn \
	 --deploy-mode client \
{'fraction_non_alphanumeric': 0.14666666666666667, 'fraction_numerical': 0.011228070175438596, 'mean_word_length': 3.9686411149825784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2526031', 'n_tokens_mistral': 563, 'n_tokens_neox': 504, 'n_words': 69}