<commit_before>#!/bin/sh
#BSUB -J sparkpi_job
#BSUB -W 00:10 # requesting 10 minutes
#BSUB -o sparkpi.log
#BSUB -e sparkpi.err
#BSUB -cwd "$(pwd)" # set the directory to the current one
#BSUB -n 24 # requesting 24 cores -- request in multiples of 24 to get entire nodes

module load new
module load java
module load open_mpi

# setup the spark path
. setup_spark.sh

# initialize the nodes
python start_spark_euler.py -c 8 -m 20g 

# creates the slaves file, starts the spark master and worker processes using mpirun
# the "-c" option specifies number of cores per worker
# -m specifies SPARK_MEMORY

# the specific example runs spark's pi estimation with a slices = 100 (first and only argument)

echo " Master is set as $HOSTNAME"

$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi --master \
    spark://$HOSTNAME:7077 $SPARK_HOME/lib/spark-examples-1.1.0-hadoop2.4.0.jar \
    100

SPARK_SLAVES=$HOME/slaves_$LSB_JOBID
$SPARK_HOME/sbin/stop-all.sh
$SPARK_HOME/sbin/stop-slaves.sh<commit_msg>fixed paths and removed cwd which is apparently done automatically<commit_after>#!/bin/sh
#BSUB -J sparkpi_job
#BSUB -W 00:10 # requesting 10 minutes
#BSUB -oo sparkpi.log # output extra o means overwrite
#BSUB -eo sparkpi.err
#BSUB -R "rusage[scratch=1000]" # 1000MB per core
#BSUB -n 24 # requesting 24 cores -- request in multiples of 24 to get entire nodes

module load new
module load java
module load open_mpi

# setup the spark path
. ./setup_spark.sh

# initialize the nodes
python start_spark_euler.py -c 8 -m 20g 

# creates the slaves file, starts the spark master and worker processes using mpirun
# the "-c" option specifies number of cores per worker
# -m specifies SPARK_MEMORY

# the specific example runs spark's pi estimation with a slices = 100 (first and only argument)

echo " Master is set as $HOSTNAME"
JARFILE="$SPARK_HOME/lib/spark-examples*.jar" # version depends on spark version

$SPARK_HOME/bin/spark-submit --class org.apache.spark.examples.SparkPi --master \
    spark://$HOSTNAME:7077 $JARFILE \
    1000

SPARK_SLAVES=$HOME/slaves_$LSB_JOBID
$SPARK_HOME/sbin/stop-all.sh
$SPARK_HOME/sbin/stop-slaves.sh
{'fraction_non_alphanumeric': 0.09344490934449093, 'fraction_numerical': 0.030218503021850304, 'mean_word_length': 4.087470449172577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2377677', 'n_tokens_mistral': 805, 'n_tokens_neox': 715, 'n_words': 255}