<commit_before>NAME="spark-master"
HOST_IP=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
docker stop $NAME
docker rm $NAME
__image="jupyter/all-spark-notebook"
#-p 4040:4040 -p 6066:6066 -p 7077:7077 -p 8080:8080
        #-p 1-65535:1-65535 \
docker run \
	--name $NAME \
	-d -t \
	--net=host \
	-u 0 \
	-p 4040:4040 -p 6066:6066 -p 7077:7077 -p 8080:8080 \
	--expose=1-65535 \
	-e SPARK_MASTER_IP=$HOST_IP \
	$__image /bin/bash
docker exec -d $NAME /usr/local/spark/sbin/start-master.sh
docker exec -d $NAME tail -n 1000 -F /usr/local/spark/logs/*.out

echo "The spark ui can be seen in a web browser at: http://"$HOST_IP":8080"
echo "run the worker on each worker node via..."
echo "bash start-worker.sh spark://"$HOST_IP":7077"

#HOST_IP=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
#export SPARK_LOCAL_IP=$HOST_IP
#export SPARK_MASTER_IP=$HOST_IP
#/usr/share/spark/spark-1.6.0-bin-hadoop2.4/sbin/start-master.sh
<commit_msg>added max_cores_per_user argument to start-master.sh<commit_after>NAME="spark-master"
HOST_IP=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
docker stop $NAME
docker rm $NAME
__image="jupyter/all-spark-notebook"

max_cores_per_user=$1
if [ "$max_cores_per_user" == "" ]; then
    max_cores_per_user="1000000000"
fi

docker run \
	--name $NAME \
	-d -t \
	--net=host \
	-u 0 \
	-p 4040:4040 -p 6066:6066 -p 7077:7077 -p 8080:8080 \
	--expose=1-65535 \
	-e SPARK_MASTER_IP=$HOST_IP \
	-e SPARK_MASTER_OPTS="-Dspark.deploy.defaultCores="$max_cores_per_user \
	$__image /bin/bash
docker exec -d $NAME /usr/local/spark/sbin/start-master.sh
docker exec -d $NAME tail -n 1000 -F /usr/local/spark/logs/*.out

echo "The spark ui can be seen in a web browser at: http://"$HOST_IP":8080"
echo "run the worker on each worker node via..."
echo "bash start-worker.sh spark://"$HOST_IP":7077"


{'fraction_non_alphanumeric': 0.16382627817482134, 'fraction_numerical': 0.09565695437053326, 'mean_word_length': 3.55, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5474779', 'n_tokens_mistral': 896, 'n_tokens_neox': 752, 'n_words': 191}