<commit_before>#!/usr/bin/env bash
#
# Bootstrap docker and accumulo on EMR cluster
#

IMAGE=quay.io/geodocker/accumulo:${TAG:-"latest"}
ACCUMULO_SECRET=DEFAULT
ACCUMULO_PASSWORD=secret
INSTANCE_NAME=accumulo
ARGS=$@

for i in "$@"
do
    case $i in
        --continue)
            CONTINUE=true
            shift
            ;;
        -i=*|--image=*)
            IMAGE="${i#*=}"
            shift
            ;;
        -s=*|--accumulo-secret=*)
            ACCUMULO_SECRET="${i#*=}"
            shift
            ;;
        -p=*|--accumulo-password=*)
            ACCUMULO_PASSWORD="${i#*=}"
            shift
            ;;
        -n=*|--instance-name=*)
            INSTANCE_NAME="${i#*=}"
            shift
            ;;

        *)
            ;;
    esac
done

# Parses a configuration file put in place by EMR to determine the role of this node
is_master() {
  if [ $(jq '.isMaster' /mnt/var/lib/info/instance.json) = 'true' ]; then
    return 0
  else
    return 1
  fi
}

### MAIN ####

# EMR bootstrap runs before HDFS or YARN are initilized
if [ ! $CONTINUE ]; then
    sudo yum -y install docker
    sudo usermod -aG docker hadoop
    sudo service docker start

    THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")"
	  TIMEOUT= is_master && TIMEOUT=3 || TIMEOUT=4
	  echo "bash -x $THIS_SCRIPT --continue $ARGS > /tmp/geodocker-accumulo-bootstrap.log" | at now + $TIMEOUT min
	  exit 0 # Bail and let EMR finish initializing
fi

YARN_RM=$(xmllint --xpath "//property[name='yarn.resourcemanager.hostname']/value/text()"  /etc/hadoop/conf/yarn-site.xml)
DOCKER_ENV="-e USER=hadoop \
-e HADOOP_MASTER_ADDRESS=$YARN_RM \
-e ZOOKEEPERS=$YARN_RM \
-e ACCUMULO_SECRET=$ACCUMULO_SECRET \
-e ACCUMULO_PASSWORD=$<PASSWORD> \
-e INSTANCE_NAME=$INSTANCE_NAME"

DOCKER_OPT="-d --net=host --restart=always"
if is_master ; then
    docker run $DOCKER_OPT --name=accumulo-master $DOCKER_ENV $IMAGE master --auto-init<commit_msg>volume mount hadoop config dir on EMR<commit_after>#!/usr/bin/env bash
#
# Bootstrap docker and accumulo on EMR cluster
#

IMAGE=quay.io/geodocker/accumulo:${TAG:-"latest"}
ACCUMULO_SECRET=DEFAULT
ACCUMULO_PASSWORD=<PASSWORD>
INSTANCE_NAME=accumulo
ARGS=$@

for i in "$@"
do
    case $i in
        --continue)
            CONTINUE=true
            shift
            ;;
        -i=*|--image=*)
            IMAGE="${i#*=}"
            shift
            ;;
        -s=*|--accumulo-secret=*)
            ACCUMULO_SECRET="${i#*=}"
            shift
            ;;
        -p=*|--accumulo-password=*)
            ACCUMULO_PASSWORD="${i#*=}"
            shift
            ;;
        -n=*|--instance-name=*)
            INSTANCE_NAME="${i#*=}"
            shift
            ;;

        *)
            ;;
    esac
done

# Parses a configuration file put in place by EMR to determine the role of this node
is_master() {
  if [ $(jq '.isMaster' /mnt/var/lib/info/instance.json) = 'true' ]; then
    return 0
  else
    return 1
  fi
}

### MAIN ####

# EMR bootstrap runs before HDFS or YARN are initilized
if [ ! $CONTINUE ]; then
    sudo yum -y install docker
    sudo usermod -aG docker hadoop
    sudo service docker start

    THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")"
	  TIMEOUT= is_master && TIMEOUT=3 || TIMEOUT=4
	  echo "bash -x $THIS_SCRIPT --continue $ARGS > /tmp/geodocker-accumulo-bootstrap.log" | at now + $TIMEOUT min
	  exit 0 # Bail and let EMR finish initializing
fi

YARN_RM=$(xmllint --xpath "//property[name='yarn.resourcemanager.hostname']/value/text()"  /etc/hadoop/conf/yarn-site.xml)
DOCKER_ENV="-e USER=hadoop \
-e ZOOKEEPERS=$YARN_RM \
-e ACCUMULO_SECRET=$ACCUMULO_SECRET \
-e ACCUMULO_PASSWORD=$<PASSWORD> \
-e INSTANCE_NAME=$INSTANCE_NAME \
-v /etc/hadoop/conf:/etc/hadoop/conf"

DOCKER_OPT="-d --net=host --restart=always"
if is_master ; then
    docker run $DOCKER_OPT --name=accumulo-master $DOCKER_ENV $IMAGE master --auto-init
{'fraction_non_alphanumeric': 0.14462490332559938, 'fraction_numerical': 0.0030935808197989174, 'mean_word_length': 2.4124890061565525, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5090011', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1368, 'n_words': 305}