<commit_before>#!/bin/bash

export verbose=

for param in "$@"
do
  case "$param" in
    --verbose) export verbose="--verbose"
    ;;
  esac
done

# setup Hadoop and Spark versions
export SPARK_VERSION=1.6.1
export HADOOP_VERSION=2.6
echo "[run-samples.sh] SPARK_VERSION=$SPARK_VERSION, HADOOP_VERSION=$HADOOP_VERSION"

export FWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# download runtime dependencies: spark
export TOOLS_DIR="$FWDIR/../tools"
[ ! -d "$TOOLS_DIR" ] && mkdir "$TOOLS_DIR"

export SPARK=spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION
export SPARK_HOME="$TOOLS_DIR/$SPARK"
if [ ! -d "$SPARK_HOME" ];
then
  wget "http://www.us.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK.tgz" -O "$TOOLS_DIR/$SPARK.tgz"
  tar xfz "$TOOLS_DIR/$SPARK.tgz" -C "$TOOLS_DIR"

  # hack: use a customized spark
  # TODO: fix the C# Worker
  export SPARK_SRC="$TOOLS_DIR/spark-$SPARK_VERSION"
  wget "http://www.us.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION.tgz" -O "$SPARK_SRC.tgz"
  tar xfz "$SPARK_SRC.tgz" -C "$TOOLS_DIR"
  pushd "$SPARK_SRC"
  sed -i "s/val useDaemon = /val useDaemon = false \/\//g" "core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala"
  build/mvn -Pyarn -Phadoop-$HADOOP_VERSION -DskipTests package 2>&1 | grep warn
  [ $? -ne 0 ] && exit 1
  cp assembly/target/scala-2.10/spark-assembly*hadoop*.jar "$SPARK_HOME/lib/"
  popd
fi
export PATH="$SPARK_HOME/bin:$PATH"

# update spark verbose mode
if [ ! "$verbose" = "--verbose" ];
then
  # redirect the logs from console (default) to /tmp
  cp "$FWDIR"/spark.conf/*.properties "$SPARK_HOME/conf/"
  sed -i "s/\${env:TEMP}/\/tmp/g" "$SPARK_HOME/conf/log4j.properties"
else
  # remove customized log4j.properties, revert back to out-of-the-box Spark logging to console
  rm -f "$SPARK_HOME"/conf/*.properties
fi<commit_msg>Implement a Daemon process to take charge of launching Worker processes<commit_after>#!/bin/bash

export verbose=

for param in "$@"
do
  case "$param" in
    --verbose) export verbose="--verbose"
    ;;
  esac
done

# setup Hadoop and Spark versions
export SPARK_VERSION=1.6.1
export HADOOP_VERSION=2.6
echo "[run-samples.sh] SPARK_VERSION=$SPARK_VERSION, HADOOP_VERSION=$HADOOP_VERSION"

export FWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# download runtime dependencies: spark
export TOOLS_DIR="$FWDIR/../tools"
[ ! -d "$TOOLS_DIR" ] && mkdir "$TOOLS_DIR"

export SPARK=spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION
export SPARK_HOME="$TOOLS_DIR/$SPARK"
if [ ! -d "$SPARK_HOME" ];
then
  wget "http://www.us.apache.org/dist/spark/spark-$SPARK_VERSION/$SPARK.tgz" -O "$TOOLS_DIR/$SPARK.tgz"
  tar xfz "$TOOLS_DIR/$SPARK.tgz" -C "$TOOLS_DIR"
fi
export PATH="$SPARK_HOME/bin:$PATH"

# update spark verbose mode
if [ ! "$verbose" = "--verbose" ];
then
  # redirect the logs from console (default) to /tmp
  cp "$FWDIR"/spark.conf/*.properties "$SPARK_HOME/conf/"
  sed -i "s/\${env:TEMP}/\/tmp/g" "$SPARK_HOME/conf/log4j.properties"
else
  # remove customized log4j.properties, revert back to out-of-the-box Spark logging to console
  rm -f "$SPARK_HOME"/conf/*.properties
fi
{'fraction_non_alphanumeric': 0.16347992351816443, 'fraction_numerical': 0.007329509241555131, 'mean_word_length': 4.33843537414966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530153', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1164, 'n_words': 263}