<commit_before>#!/bin/bash

# script parameters are
# param 1 - number of servers to add.
# param 2 - access key 
# param 3 - secret key
# param 4 - DNS of the master node
set +x
COUNT=$1
ACCESS_KEY=$2
SECRET_KEY=$3
MASTER_NODE=$4
S3_BUCKET=$5
if [ -z $S3_BUCKET ];
then
	echo usage:
	echo "$0 <COUNT> <ACCESS_KEY> <SECRET_KEY> <MASTER_NODE_DNS> <S3_BUCKET>"
	exit 2
fi
#TODO: Create profile with keys
#TODO: Fixup logging
BATCH_ID=$(uuidgen)
. resize.config
LOG=${LOG_DIR}/${BATCH_ID}.log
LOG_APPEND='tee -a $LOG'
take_lock
echo trying to start cluster $BATCH_ID|$LOG_APPEND
store_cluster_id $BATCH_ID
echo creating security group if required|$LOG_APPEND
. create_impala_security_group.sh
SECURITY_GROUP_IDS=$(get_or_create_security_group ${SECURITY_GROUP})
echo checking if requested AMI: $IMAGE_ID available|$LOG_APPEND
IMAGE_AVAILABLE=$($AWS_CMD describe-images --image-ids $IMAGE_ID|grep -o $IMAGE_ID)
if [ -z "$IMAGE_AVAILABLE" ];
then
	echo Image: $IMAGE_ID is not available
	echo please check configuration.
	exit 1
fi
echo requesting to start $COUNT instances of $INSTANCE_TYPE size with AMI: $IMAGE_ID

$AWS_CMD run-instances --image-id $IMAGE_ID --count $COUNT --instance-type $INSTANCE_TYPE --security-group-ids $SECURITY_GROUP_IDS --placement AvailabilityZone=$AVAILABILITY_ZONE --key-name $KEY_NAME --client-token $BATCH_ID --user-data "\'$USER_DATA\'" >> $LOG

echo run-instances request sent, waiting for all instances to run

$AWS_CMD wait instance-running --filters Name=client-token,Values=$BATCH_ID

echo all instances running querying instance details 
TMP_FILE=/tmp/${BATCH_ID}.info
$AWS_CMD describe-instances --filters Name=client-token,Values=$BATCH_ID >$TMP_FILE
echo getting DNS names
DNS_NAMES=$(grep $BATCH_ID <${TMP_FILE}|cut -f 14|tee ${CLUSTER_HOSTS})

echo Adding ssh public-keys records to $SSH_KNOWN_HOSTS_FILE
echo -n " " >$SSH_KNOWN_HOSTS_FILE
#TODO: Add timeout
#TODO: There is still a possible bug when all hosts does not awaik togather,
#so when each host exposes more than one key, key count will be greater than required hosts but not enough to connect them all
while [ $(cat $SSH_KNOWN_HOSTS_FILE|wc -l) -lt $(echo $DNS_NAMES|wc -w) ]; do
	sleep 20
	echo trying to perform keyscan
	#clear known hosts before attempt
	echo -n  " " >$SSH_KNOWN_HOSTS_FILE
	for host in $DNS_NAMES; do
		ssh-keyscan -H $host >>$SSH_KNOWN_HOSTS_FILE
	done<commit_msg>Keyscan optimization - wait less when instances are fast<commit_after>#!/bin/bash

# script parameters are
# param 1 - number of servers to add.
# param 2 - access key 
# param 3 - secret key
# param 4 - DNS of the master node
set +x
COUNT=$1
ACCESS_KEY=$2
SECRET_KEY=$3
MASTER_NODE=$4
S3_BUCKET=$5
if [ -z $S3_BUCKET ];
then
	echo usage:
	echo "$0 <COUNT> <ACCESS_KEY> <SECRET_KEY> <MASTER_NODE_DNS> <S3_BUCKET>"
	exit 2
fi
#TODO: Create profile with keys
#TODO: Fixup logging
BATCH_ID=$(uuidgen)
. resize.config
LOG=${LOG_DIR}/${BATCH_ID}.log
LOG_APPEND='tee -a $LOG'
take_lock
echo trying to start cluster $BATCH_ID|$LOG_APPEND
store_cluster_id $BATCH_ID
echo creating security group if required|$LOG_APPEND
. create_impala_security_group.sh
SECURITY_GROUP_IDS=$(get_or_create_security_group ${SECURITY_GROUP})
echo checking if requested AMI: $IMAGE_ID available|$LOG_APPEND
IMAGE_AVAILABLE=$($AWS_CMD describe-images --image-ids $IMAGE_ID|grep -o $IMAGE_ID)
if [ -z "$IMAGE_AVAILABLE" ];
then
	echo Image: $IMAGE_ID is not available
	echo please check configuration.
	exit 1
fi
echo requesting to start $COUNT instances of $INSTANCE_TYPE size with AMI: $IMAGE_ID

$AWS_CMD run-instances --image-id $IMAGE_ID --count $COUNT --instance-type $INSTANCE_TYPE --security-group-ids $SECURITY_GROUP_IDS --placement AvailabilityZone=$AVAILABILITY_ZONE --key-name $KEY_NAME --client-token $BATCH_ID --user-data "\'$USER_DATA\'" >> $LOG

echo run-instances request sent, waiting for all instances to run

$AWS_CMD wait instance-running --filters Name=client-token,Values=$BATCH_ID

echo all instances running querying instance details 
TMP_FILE=/tmp/${BATCH_ID}.info
$AWS_CMD describe-instances --filters Name=client-token,Values=$BATCH_ID >$TMP_FILE
echo getting DNS names
DNS_NAMES=$(grep $BATCH_ID <${TMP_FILE}|cut -f 14|tee ${CLUSTER_HOSTS})

echo Adding ssh public-keys records to $SSH_KNOWN_HOSTS_FILE
echo -n " " >$SSH_KNOWN_HOSTS_FILE
#TODO: Add timeout
#TODO: There is still a possible bug when all hosts does not awaik togather,
#so when each host exposes more than one key, key count will be greater than required hosts but not enough to connect them all
while [ $(cat $SSH_KNOWN_HOSTS_FILE|wc -l) -lt $(echo $DNS_NAMES|wc -w) ]; do
	echo trying to perform keyscan
	#clear known hosts before attempt
	echo -n  " " >$SSH_KNOWN_HOSTS_FILE
	for host in $DNS_NAMES; do
		ssh-keyscan -H $host >>$SSH_KNOWN_HOSTS_FILE &
	done
	wait
{'fraction_non_alphanumeric': 0.10338384886859041, 'fraction_numerical': 0.007473531243512559, 'mean_word_length': 5.289817232375979, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6521598', 'n_tokens_mistral': 1728, 'n_tokens_neox': 1689, 'n_words': 535}