<commit_before>#!/bin/bash

# simple script for delaying job updates until the cluster is green
# this lets us do kind of a rolling restart of elasticsearch nodes and avoid
# noticeable downtime
#
# this uses two state files
# 1) to store which task we're working on (update, shutdown)
# 2) whether we've done a one-time handling of that task
#
# this is necessary because we don't want to make changes to the cluster until
# it is green (which may not be the first time this script is invoked)
#
# this is obviously very simplistic... if/when you need to get fancy with your
# cluster, you'll want to disable this for more advanced allocation and host
# management tasks
#
# this is invoked somewhere around here:
# https://github.com/cloudfoundry/bosh/blob/7a7e42312a6a1dce50b578be579f17d10797e556/bosh_agent/lib/bosh_agent/message/drain.rb#L172

<% if !p('elasticsearch.drain') %>
# elasticsearch.drain is disabled in the manifest
echo "0"
exit 0
<% elsif !p('elasticsearch.node.allow_data') %>
# this node does not contain data, so we can skip all these data-related tasks
echo "0"
exit 0
<% end %>

JOB_STATUS=$1

STATE_JOB_STATUS=/var/vcap/jobs/elasticsearch/config/.boshdrain
STATE_HANDLED=/var/vcap/jobs/elasticsearch/config/.boshdrainhandled

function proceed {
    [ ! -f $STATE_JOB_STATUS ] || rm $STATE_JOB_STATUS
    [ ! -f $STATE_HANDLED ] || rm $STATE_HANDLED

    echo "0"
    exit 0
}

if [ "job_check_status" != "$JOB_STATUS" ] ; then
    echo $JOB_STATUS > $STATE_JOB_STATUS
fi

set -e # exit immediately if a simple command exits with a non-zero status
set -u # report the usage of uninitialized variables

if ! nc -zw 8 localhost 9300 2>&1 > /dev/null ; then
    # elasticsearch transport isn't already online, so there should be no harm
    # in going offline

    proceed
elif [ "green" == `wget -qO- --timeout 16 'localhost:9200/_cat/health?h=status'` ] ; then
    # cluster is green, so let's proceed

    JOB_STATUS=$(cat $STATE_JOB_STATUS)

    if [ ! -f $STATE_HANDLED ] ; then
        touch $STATE_HANDLED

        if [ "job_changed" == "$JOB_STATUS" ] || [ "job_unchanged" == "$JOB_STATUS" ] ; then
            # we should disable allocations to avoid indices getting reinitialized
            # from peer shards while we're offline
            curl -sX PUT -d '{"transient":{"cluster.routing.allocation.enable":"primaries"}}' 'localhost:9200/_cluster/settings' > /dev/null

            # during startup, we'll look for this value to know if we should
            # restore the normal allocation settings
            curl -sX PUT -d '{"drained":1}' 'localhost:9200/.bosh/drain/<%= name %>--<%= index %>' > /dev/null
        elif [ "job_shutdown" == "$JOB_STATUS" ] ; then
            curl -sX PUT -d '{"transient":{"cluster.routing.allocation.exclude._name":"<%= name %>/<%= index %>"}}' 'localhost:9200/_cluster/settings' > /dev/null

            # come back soon to check on the rebalancing status
            echo "-10"
            exit 0
        fi
    fi

    if [ "job_shutdown" == "$JOB_STATUS" ] ; then
        # since we're terminating, we really care about whether we're still
        # responsible for any shards

        if [ "0" == `wget -qO- 'localhost:9200/_cat/shards?h=index,shard,node' | grep '<%= name %>/<%= index %>' | wc -l` ] ; then
            # this node has no more shards allocated to it, so it's safe to
            # shutdown

            proceed
        fi

        echo "-60"
        exit 0
    fi

    proceed
fi

# check back again soon to see if the cluster is green
echo "-10"
exit 0
<commit_msg>settle on curl and subshells /issue #51<commit_after>#!/bin/bash

# simple script for delaying job updates until the cluster is green
# this lets us do kind of a rolling restart of elasticsearch nodes and avoid
# noticeable downtime
#
# this uses two state files
# 1) to store which task we're working on (update, shutdown)
# 2) whether we've done a one-time handling of that task
#
# this is necessary because we don't want to make changes to the cluster until
# it is green (which may not be the first time this script is invoked)
#
# this is obviously very simplistic... if/when you need to get fancy with your
# cluster, you'll want to disable this for more advanced allocation and host
# management tasks
#
# this is invoked somewhere around here:
# https://github.com/cloudfoundry/bosh/blob/7a7e42312a6a1dce50b578be579f17d10797e556/bosh_agent/lib/bosh_agent/message/drain.rb#L172

<% if !p('elasticsearch.drain') %>
# elasticsearch.drain is disabled in the manifest
echo "0"
exit 0
<% elsif !p('elasticsearch.node.allow_data') %>
# this node does not contain data, so we can skip all these data-related tasks
echo "0"
exit 0
<% end %>

JOB_STATUS=$1

STATE_JOB_STATUS=/var/vcap/jobs/elasticsearch/config/.boshdrain
STATE_HANDLED=/var/vcap/jobs/elasticsearch/config/.boshdrainhandled

function proceed {
    [ ! -f $STATE_JOB_STATUS ] || rm $STATE_JOB_STATUS
    [ ! -f $STATE_HANDLED ] || rm $STATE_HANDLED

    echo "0"
    exit 0
}

if [ "job_check_status" != "$JOB_STATUS" ] ; then
    echo $JOB_STATUS > $STATE_JOB_STATUS
fi

set -e # exit immediately if a simple command exits with a non-zero status
set -u # report the usage of uninitialized variables

if ! nc -zw 8 localhost 9300 2>&1 > /dev/null ; then
    # elasticsearch transport isn't already online, so there should be no harm
    # in going offline

    proceed
elif [ "green" == $(curl -s --max-time 16 'localhost:9200/_cat/health?h=status') ] ; then
    # cluster is green, so let's proceed

    JOB_STATUS=$(cat $STATE_JOB_STATUS)

    if [ ! -f $STATE_HANDLED ] ; then
        touch $STATE_HANDLED

        if [ "job_changed" == "$JOB_STATUS" ] || [ "job_unchanged" == "$JOB_STATUS" ] ; then
            # we should disable allocations to avoid indices getting reinitialized
            # from peer shards while we're offline
            curl -sX PUT -d '{"transient":{"cluster.routing.allocation.enable":"primaries"}}' 'localhost:9200/_cluster/settings' > /dev/null

            # during startup, we'll look for this value to know if we should
            # restore the normal allocation settings
            curl -sX PUT -d '{"drained":1}' 'localhost:9200/.bosh/drain/<%= name %>--<%= index %>' > /dev/null
        elif [ "job_shutdown" == "$JOB_STATUS" ] ; then
            curl -sX PUT -d '{"transient":{"cluster.routing.allocation.exclude._name":"<%= name %>/<%= index %>"}}' 'localhost:9200/_cluster/settings' > /dev/null

            # come back soon to check on the rebalancing status
            echo "-10"
            exit 0
        fi
    fi

    if [ "job_shutdown" == "$JOB_STATUS" ] ; then
        # since we're terminating, we really care about whether we're still
        # responsible for any shards

        if [ "0" == $(curl -s --max-time 32 'localhost:9200/_cat/shards?h=index,shard,node' | grep '<%= name %>/<%= index %>' | wc -l) ] ; then
            # this node has no more shards allocated to it, so it's safe to
            # shutdown

            proceed
        fi

        echo "-60"
        exit 0
    fi

    proceed
fi

# check back again soon to see if the cluster is green
echo "-10"
exit 0

{'fraction_non_alphanumeric': 0.11115765954475632, 'fraction_numerical': 0.02262253875157101, 'mean_word_length': 2.9656699889258027, 'pattern_counts': {'":': 10, '<': 21, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2128303', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2205, 'n_words': 786}