<commit_before>#!/bin/bash

export ROOT_DIR=$( cd "$( dirname "$BASH_SOURCE" )" && pwd )/..

for env in $ROOT_DIR/cfg/*.env; do if [ -f "$env" ]; then . "$env"; fi; done

set -x -e

WAIT_TASK=${1:-"true"}
DELETE_CLUSTER=${2:-"false"}
PROXY_CONNECT=${3:-"false"}
CLUSTER_NAME=${4:-"$CLUSTER_NAME"}
SERVICE_TYPE=${5:-"$CLUSTER_SERVICE_TYPE"}
WORKERS_NUMBER=${6:-"$CLUSTER_WORKERS_NUMBER"}
INSTANCE_TYPE=${7:-"$CLUSTER_INSTANCE_TYPE"}
CDH_VERSION=${8:-"$CLUSTER_CDH_VERSION"}
ALTUS_ENV=${9:-"$CLUSTER_ENVIRONMENT"}
SSH_KEY=${10:-"~/.ssh/director"}
MANAGER_SERVER_USER=${11:-"cmuser"}
MANAGER_SERVER_PWORD=${12:-"<PASSWORD>"}

BOOTSTRAP_FILE=$ROOT_DIR/bin/cldr-provision-altus-bootstrap.sh

[[ ! -f "$SSH_KEY" ]] && SSH_KEY="$ROOT_DIR/cfg/provision"

if [ "$DELETE_CLUSTER" = "true" ]; then
  if [ $(altus dataeng list-clusters --cluster-names "$CLUSTER_NAME" 2>&1 | grep status | grep CREATED | wc -l) -eq 1 ]; then
    altus dataeng delete-cluster --cluster-name="$CLUSTER_NAME"
  fi
else
  if [ $(altus dataeng list-clusters --cluster-names "$CLUSTER_NAME" 2>&1 | grep "No cluster found" | wc -l) -ne 0 ]; then
    echo "virtualenv /tmp/pyspark-env" > $BOOTSTRAP_FILE
    if [ -f $ROOT_DIR/lib/python/conda.yml ]; then
      LIB_PIPS=($(grep = $ROOT_DIR/lib/python/conda.yml))
      for LIB_PIP in "${LIB_PIPS[@]}"; do
        [[ "$LIB_PIP" != "-" ]] && \
          [[ $(echo $LIB_PIP | grep "pyspark=" | wc -l) -eq 0 ]] && \
          [[ $(echo $LIB_PIP | grep "python=" | wc -l) -eq 0 ]] && \
          echo "/tmp/pyspark-env/bin/pip install "$(echo "$LIB_PIP" | sed 's/=/==/g') >> $BOOTSTRAP_FILE
      done
    fi
    [[ ! -f "$SSH_KEY" ]] && ssh-keygen -N '' -f "$SSH_KEY"
    altus dataeng create-aws-cluster \
      --service-type="$SERVICE_TYPE" \
      --workers-group-size="$WORKERS_NUMBER" \<commit_msg>Update altus key management<commit_after>#!/bin/bash

export ROOT_DIR=$( cd "$( dirname "$BASH_SOURCE" )" && pwd )/..

for env in $ROOT_DIR/cfg/*.env; do if [ -f "$env" ]; then . "$env"; fi; done

set -x -e

WAIT_TASK=${1:-"true"}
DELETE_CLUSTER=${2:-"false"}
PROXY_CONNECT=${3:-"false"}
CLUSTER_NAME=${4:-"$CLUSTER_NAME"}
SERVICE_TYPE=${5:-"$CLUSTER_SERVICE_TYPE"}
WORKERS_NUMBER=${6:-"$CLUSTER_WORKERS_NUMBER"}
INSTANCE_TYPE=${7:-"$CLUSTER_INSTANCE_TYPE"}
CDH_VERSION=${8:-"$CLUSTER_CDH_VERSION"}
ALTUS_ENV=${9:-"$CLUSTER_ENVIRONMENT"}
SSH_KEY=${10:-"/Users/graham/.ssh/director"}
MANAGER_SERVER_USER=${11:-"cmuser"}
MANAGER_SERVER_PWORD=${12:-"<PASSWORD>"}

BOOTSTRAP_FILE=$ROOT_DIR/bin/cldr-provision-altus-bootstrap.sh

[[ ! -f "$SSH_KEY" ]] && SSH_KEY="$ROOT_DIR/cfg/provision"

if [ "$DELETE_CLUSTER" = "true" ]; then
  if [ $(altus dataeng list-clusters --cluster-names "$CLUSTER_NAME" 2>&1 | grep status | grep CREATED | wc -l) -eq 1 ]; then
    altus dataeng delete-cluster --cluster-name="$CLUSTER_NAME"
  fi
else
  if [ $(altus dataeng list-clusters --cluster-names "$CLUSTER_NAME" 2>&1 | grep "No cluster found" | wc -l) -ne 0 ]; then
    echo "virtualenv /tmp/pyspark-env" > $BOOTSTRAP_FILE
    if [ -f $ROOT_DIR/lib/python/conda.yml ]; then
      LIB_PIPS=($(grep = $ROOT_DIR/lib/python/conda.yml))
      for LIB_PIP in "${LIB_PIPS[@]}"; do
        [[ "$LIB_PIP" != "-" ]] && \
          [[ $(echo $LIB_PIP | grep "pyspark=" | wc -l) -eq 0 ]] && \
          [[ $(echo $LIB_PIP | grep "python=" | wc -l) -eq 0 ]] && \
          echo "/tmp/pyspark-env/bin/pip install "$(echo "$LIB_PIP" | sed 's/=/==/g') >> $BOOTSTRAP_FILE
      done
    fi
    [[ ! -f "$SSH_KEY" ]] && ssh-keygen -N '' -f "$SSH_KEY"
    altus dataeng create-aws-cluster \
      --service-type="$SERVICE_TYPE" \
      --workers-group-size="$WORKERS_NUMBER" \
{'fraction_non_alphanumeric': 0.21276013143483022, 'fraction_numerical': 0.012595837897042717, 'mean_word_length': 3.6713554987212276, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5328374', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1488, 'n_words': 269}