<commit_before>#!/bin/sh

default_url=http:$ETCD_IP:$ETCD_PORT/v2/keys/etcd_spark

# load config file for master
drivers=0
workers=0
declare -A master_config

cat master/config | while read key value; do
    master_config[$key]=$value
done

drivers=${master_config["drivers"]}
workers=${master_config["workers"]}

curl -L $default_url/${master_config["name"]}/name -XPUT -d value=${master_config["name"]}
curl -L $default_url/${master_config["name"]}/spark_env -XPUT --data-urlencode value@master/spark-env.sh
curl -L $default_url/${master_config["name"]}/log4j -XPUT --data-urlencode value@master/log4j.properties

for (( i=1; i<=$drivers ; i++ ))
do
    curl -L $default_url/${master_config["name"]}/driver$i/spark_env -XPUT --data-urlencode value@driver$i/spark-env.sh
done

for (( i=1; i<=$workers ; i++ ))
do
    curl -L $default_url/${master_config["name"]}/worker$i/spark_env -XPUT --data-urlencode value@worker$i/spark-env.sh
done
<commit_msg>fixed laod_config.sh<commit_after>#!/bin/sh

default_url=http:$ETCD_IP:$ETCD_PORT/v2/keys/etcd_spark

drivers=0
workers=0

# load config file for master
cat master/config | {
    declare -A master_config
    while read key value; do
        master_config[$key]=$value
    done

    drivers=${master_config["drivers"]}
    workers=${master_config["workers"]}

    curl -L $default_url/${master_config["name"]}/name -XPUT -d value=${master_config["name"]}
    curl -L $default_url/${master_config["name"]}/spark_env -XPUT --data-urlencode value@master/spark-env.sh
    curl -L $default_url/${master_config["name"]}/log4j -XPUT --data-urlencode value@master/log4j.properties

    for (( i=1; i<=$drivers ; i++ ))
    do
        curl -L $default_url/${master_config["name"]}/driver$i/spark_env -XPUT --data-urlencode value@driver$i/spark-env.sh
    done

    for (( i=1; i<=$workers ; i++ ))
    do
        curl -L $default_url/${master_config["name"]}/worker$i/spark_env -XPUT --data-urlencode value@worker$i/spark-env.sh
    done
}

{'fraction_non_alphanumeric': 0.17213528520949015, 'fraction_numerical': 0.007067137809187279, 'mean_word_length': 4.460055096418733, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3179217', 'n_tokens_mistral': 743, 'n_tokens_neox': 718, 'n_words': 131}