<commit_before>#!/usr/bin/env bash

# Directory where spark-submit is defined
# Install spark from https://spark.apache.org/downloads.html
#
# Works in mac
# TODO: fix it for linux
USER=$(whoami)
SPARK_HOME=/Users/${USER}/spark-2.1.1-bin-hadoop2.7

# JAR containing a simple hello world
JARFILE=`pwd`/target/scala-2.11/elasticsearch-spark-example-assembly-0.1.0.jar

# Run it locally
${SPARK_HOME}/bin/spark-submit --class org.zouzias.elasticsearch.spark.ElasticSparkHelloWorld \
    --master local \
	--packages org.elasticsearch:elasticsearch-spark-20_2.11:5.4.3 \
	 $JARFILE<commit_msg>5.x: script update<commit_after>#!/usr/bin/env bash

# Directory where spark-submit is defined
# Install spark from https://spark.apache.org/downloads.html
#
# Works in mac
# TODO: fix it for linux
USER=$(whoami)
SPARK_HOME=/Users/${USER}/spark-2.1.2-bin-hadoop2.7

# JAR containing a simple hello world
JARFILE=`pwd`/target/scala-2.11/elasticsearch-spark-example-assembly-0.1.0.jar

# Run it locally
${SPARK_HOME}/bin/spark-submit --class org.zouzias.elasticsearch.spark.ElasticSparkHelloWorld \
    --master local \
	--packages org.elasticsearch:elasticsearch-spark-20_2.11:5.4.3 \
	 $JARFILE
{'fraction_non_alphanumeric': 0.1437024513947591, 'fraction_numerical': 0.03296703296703297, 'mean_word_length': 4.238938053097345, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2345528', 'n_tokens_mistral': 448, 'n_tokens_neox': 421, 'n_words': 79}