<commit_before>#!/usr/bin/env bash

# This file executes rax-autoscaler and checks whether the
# output indicates that there were autoscale nodes NOT in 
# the load balancer.
# If that happens, create /tmp/rax_autoscale_failure
# This file is monitored by fail_file_monitor.py (check
# header of this file for more information)
# 

fail_file=/tmp/rax_autoscale_failure
log_file=$(mktemp)
/usr/bin/autoscale --config /opt/autoscale/autoscale_setup/rax-autoscaler-config.json > $log_file 2>&1
grep "Consensus was to scale down" $log_file 1>/dev/null 2>&1 && cat $log_file > $fail_file

# Merge log file for this run with the usual log
cat $log_file >> /var/log/rax-autoscaler/logging.log
rm -f $log_file<commit_msg>Updated wrapper.sh to automatically handle the counter of failures and resets if issue is fixed<commit_after>#!/usr/bin/env bash

# ============================================================ #
# This file executes 'rax-autoscaler' and checks whether the
# output indicates that there is a need to 'scale down' but NOT
# all the nodes in the autoscale group are in the load balancer.
#
# This could be simply because rax-autoscaler checked while the
# new server was still building/configuring or a real issue and
# we need a way to be notified if this keeps happening for too 
# long, to avoid to get autoscale stuck with the autoscale 
# max_nodes limit reached without actually serving traffic and
# the inability to scale down.
#
# Script looks for the string "Consensus was to scale down" in
# the output of rax-autoscaler.
# If that happens, it tracks the number of failure and in case
# it goes over the number set in MAXFAILURES variable, it will
# create /tmp/rax_autoscale_failure. This file is monitored by 
# fail_file_monitor.py
# (check header of this file for more information)
# Once the status goes back to normal (scale_up, scale_down or
# do_nothing), it will reset counters and alert file.
# ============================================================ #

# Please set this variable accordingly with your setup
MAXFAILURES=3

# MAXFAILURES should be calculated in this way:
# ((av_time_new_server_ready / rax-autoscaler_cron_time)) x2 
# 
# the 'x2' is more as a precaution to allow some buffer in case
# a server requires more time. We allow DOUBLE time.
#
# Example:
# 15 minutes average time to have a server built and configured
# rax-autoscaler set in cron to run every 2 minutes
# this means => ~7-8 runs x 2 = 14-16 MAXFAILURES 
# ============================================================ #


# Define log files
FAIL_FILE=/tmp/rax_autoscale_failure
LOG_FAILURES=/tmp/rax_autoscale_failures_report
TMP_LOG_FILE=$(mktemp)
RAX_LOG_FILE=/var/log/rax-autoscaler/logging.log

touch $LOG_FAILURES
FAILS=$(cat $LOG_FAILURES)
[[ -z "$FAILS" ]] && FAILS=0

# Execute rax-autoscaler and get the output
/usr/bin/autoscale --config /opt/autoscale/autoscale_setup/rax-autoscaler-config.json > $TMP_LOG_FILE 2>&1

# Check output of rax-autoscaler:
# if "Consensus was to scale down" found => log +1 failure
# for any other output => reset all

if [[ $(grep "Consensus was to scale down" $TMP_LOG_FILE) ]]
    then
	((FAILS++))
	echo $FAILS > $LOG_FAILURES
    else 
	FAILS=0
	rm -f $LOG_FAILURES > /dev/null 2>&1
	rm -f $FAIL_FILE > /dev/null 2>&1
fi


# Generate FAIL_FILE if failures > MAXFAILURES limit
[ $FAILS -gt $MAXFAILURES ] && cat $TMP_LOG_FILE > $FAIL_FILE


# Merge output log file from this run with the cummulative rax-autoscaler log
cat $TMP_LOG_FILE >> $RAX_LOG_FILE
rm -f $TMP_LOG_FILE

{'fraction_non_alphanumeric': 0.1310872027180068, 'fraction_numerical': 0.007644394110985277, 'mean_word_length': 4.634768740031898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6235248', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1097, 'n_words': 421}