<commit_before>#!/usr/bin/env bash

#
# Copyright 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

usage() {
	echo Usage: $0 [options] [file]
    echo
	echo "options:"
	echo "   --help                      usage message"
	echo "   --master MASTER_URL         spark://host:port, mesos://host:port, yarn, or local."
	echo "   --name NAME                 A name of your application."
	
}

#
# Check for spark jar
#
if [ -z "$SPARK_HOME" ]; then
    echo "Need to set SPARK_HOME to the location of the Spark distribution"
    exit 1
fi

#
# Check for eclairJS-nashorn jar
#
if [ -z "$ECLAIRJS_JAR" ]; then
	export ECLAIRJS_JAR=./target/eclairjs-nashorn-0.1.jar
fi

# 
# Check for java 1.8
#
if type -p java > /dev/null; then
	#echo found java executable in PATH
	_java=java
elif [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]];  then
		#echo found java executable in JAVA_HOME     
		_java="$JAVA_HOME/bin/java"
else
	echo "Java 8 required, please Java 1.8.0_60 or greater."
	exit 1
fi

if [[ "$_java" ]]; then
	version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
	#echo version "$version"
	if [[ "$version" < "1.8.0_60" ]]; then
	    echo java version greater than 1.8.0_59 is required.
	    exit 1
	fi
fi


# Use > 1 to consume two arguments per pass in the loop (e.g. each
# argument has a corresponding value to go with it).
# Use > 0 to consume one or more arguments per pass in the loop (e.g.
# some arguments don't have a corresponding value to go with it such
# as in the --default example).
# note: if this is set to > 0 the /etc/hosts part is not recognized ( may be a bug )
while [[ $# > 0 ]]
do
key="$1"

case $key in
	-h|--help)
    DISPLAY_HELP="true"
	shift # past argument
	;;
    -n|--name)
    export APP_NAME="$2"
    shift # past argument
    ;;
    -m|--master)
    export MASTER="$2"
    shift #past argument
    ;;
    *)
    SCRIPT_FILE=$1
    ;;
esac
shift # past argument or value
done

if [[ -n $DISPLAY_HELP ]]; then
	usage
	exit 1
fi

#
# start the REPL
#
if [ -z "$ECLAIRJS_DEBUG" ]; then
	$_java -cp $ECLAIRJS_JAR:"$SPARK_HOME/lib/*" org.eclairjs.nashorn.SparkJS $SCRIPT_FILE
else 
	$_java -Dlog4j.configuration=file:"./src/main/resources/conf/log4j.prop" -cp $ECLAIRJS_JAR:"$SPARK_HOME/lib/*" org.eclairjs.nashorn.SparkJS $SCRIPT_FILE
fi<commit_msg>Fixes #5<commit_after>#!/usr/bin/env bash

#
# Copyright 2015 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#



#
# Check for spark jar
#
if [ -z "$SPARK_HOME" ]; then
    echo "Need to set SPARK_HOME to the location of the Spark distribution"
    exit 1
fi

#
# Check for eclairJS-nashorn jar
#
if [ -z "$ECLAIRJS_JAR" ]; then
	export ECLAIRJS_JAR=./target/eclairjs-nashorn-0.1.jar
fi

# 
# Check for java 1.8
#
if type -p java > /dev/null; then
	#echo found java executable in PATH
	_java=java
elif [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]];  then
		#echo found java executable in JAVA_HOME     
		_java="$JAVA_HOME/bin/java"
else
	echo "Java 8 required, please Java 1.8.0_60 or greater."
	exit 1
fi

if [[ "$_java" ]]; then
	version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
	#echo version "$version"
	if [[ "$version" < "1.8.0_60" ]]; then
	    echo java version greater than 1.8.0_59 is required.
	    exit 1
	fi
fi

# Use > 1 to consume two arguments per pass in the loop (e.g. each
# argument has a corresponding value to go with it).
# Use > 0 to consume one or more arguments per pass in the loop (e.g.
# some arguments don't have a corresponding value to go with it such
# as in the --default example).
# note: if this is set to > 0 the /etc/hosts part is not recognized ( may be a bug )
options=" ";
proargs=" ";
while [[ $# > 0 ]]
do
key="$1"

case $key in
	-*)
    options="$options $1 $2";
    shift # past argument
    ;;
    *)
    proargs=$1
    ;;
esac
shift # past argument or value
done

#
# start the REPL
#

exec "${SPARK_HOME}"/bin/spark-submit --class org.eclairjs.nashorn.SparkJS --name "EclairJSShell" $options $ECLAIRJS_JAR  $proargs

{'fraction_non_alphanumeric': 0.12256267409470752, 'fraction_numerical': 0.01770791882212495, 'mean_word_length': 3.144270403957131, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2305193', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1711, 'n_words': 646}