<commit_before>#!/usr/bin/env bash

CASE=$1



printf "\n\n"
echo "============================== [`date`] Configuration setup ================================="


function wait_step_event_node {
    # NOTE: we are using a large timeout because the haproxy node can be long to download & install haproxy
    opsbro gossip events wait "$1-$2" --timeout=180
    if [ $? != 0 ];then
       echo "ERROR: `date` the node $2 did not reach STEP $1"
       exit 2
    fi
    echo "NODE $2 did reach step $1 at `date`"
}

function wait_step_event {
    echo " * Waiting for other nodes events. Starts at `date`"
    wait_step_event_node $1 "NODE-HTTP-1"
    wait_step_event_node $1 "NODE-HTTP-2"
    wait_step_event_node $1 "NODE-HAPROXY"
    wait_step_event_node $1 "NODE-CLIENT"
}

echo "$CASE starts to run `date` $TRAVIS"


# Set a valid display name for debug
opsbro agent parameters set display_name "$CASE"


# Haproxy is the central node, it will ensure us that all nodes will talk togethers
if [ $CASE == "NODE-HAPROXY" ]; then
   opsbro agent parameters set proxy-node true
   opsbro agent parameters add groups demo_haproxy
fi


# Client node will need to expose nodes with DNS
if [ $CASE == "NODE-CLIENT" ]; then
   opsbro packs overload global.dns
   opsbro packs parameters set local.dns.port  53
   opsbro agent parameters add groups dns-listener
   printf "nameserver 127.0.0.1\n" > /etc/resolv.conf
fi


# We are ready, launch the whole daemons
printf "\n\n"
echo "============================== [`date`] Launching daemon ================================="
/etc/init.d/opsbro start

echo "Daemon started `date`"

# For debug purpose
ip addr show | grep 'scope global'




# Let the nodes join them selve.
# NOTE: the haproxy node will be slower to get here, because he need to install haproxy during the start
echo "Launching UDP detection `date`"
opsbro gossip detect --auto-join




printf "\n\n"
echo "============================== [`date`] STEP1 synchronization ================================="
echo "Adding my own event: STEP1-$CASE  at `date`"

opsbro gossip events add  "STEP1-$CASE"
wait_step_event "STEP1"

echo "All nodes are syncronized at `date`"





printf "\n\n"
echo "============================== [`date`] Checking we are 4 members ================================="
MEMBERS=$(opsbro gossip members)
NB_MEMBERS=$(echo "$MEMBERS" | grep 'docker-container' | wc -l)

printf "Current members: `date` $MEMBERS\n"


# Must have all 4 nodes available
if [ $NB_MEMBERS != 4 ]; then
   echo "`date` BAD number of members: $NB_MEMBERS"
   echo "$MEMBERS"
   cat /var/log/opsbro/gossip.log
   exit 2
fi


# Check that HTTP nodes are running well
if [ $CASE == "NODE-HTTP-1" ] || [ $CASE == "NODE-HTTP-2" ]; then

   printf "\n\n"
   echo "============================== [`date`] $CASE apache checking ================================="

   PAYLOAD=$(curl -s http://localhost/)
   echo "PAYLOAD $PAYLOAD"
   if [ "$PAYLOAD" != "$CASE" ]; then
      echo "ERROR: `date` the http page is not ready"
      exit 2
   fi

   # They must have the apache group
   opsbro detectors wait-group 'apache'
   if [ $? != 0 ];then
      echo "ERROR:  `date` the apache group was not detected"
      exit 2
   fi

   # Let the httproxy node know we are ready for checking the haproxy state
   opsbro gossip events add  "HTTP-READY-$CASE"

   echo "$CASE  `date` will wait for queries until all others nodes are done"

   opsbro gossip events add  "ENDING-$CASE"
   wait_step_event "ENDING"


   echo "$CASE exiting at `date`"
   exit 0
fi


# Haproxy compliance must have installed the haproxy package
if [ $CASE == "NODE-HAPROXY" ]; then

    printf "\n\n"
    echo "============================== [`date`] $CASE waiting for HTTP node to be ready ================================="

    ps axjf

    # Wait until the http-1 and http-2 are ready to be queried
    wait_step_event_node "HTTP-READY" "NODE-HTTP-1"
    wait_step_event_node "HTTP-READY" "NODE-HTTP-2"


    printf "\n\n"
    echo "============================== [`date`] $CASE checking that haproxy is happy ================================="
    echo "HAPROXY: look to see if haproxy is installed  `date`"
    opsbro compliance wait-compliant "TEST HAPROXY" --timeout=60
    if [ $? != 0 ]; then
        echo "Haproxy compliance cannot be fixed in COMPLIANCE state in 60s"
        opsbro compliance state
        exit 2
    fi

    echo "Compliance history that did install HAPROXY   `date`"
    opsbro compliance state


    echo "The haproxy group should now be detected"
    opsbro detectors wait-group 'haproxy'
    if [ $? != 0 ];then
       echo "ERROR: the haproxy group was not detected"
       exit 2
    fi


    echo "Testing local proxying"
    /etc/init.d/haproxy status | grep 'haproxy is running'
    if [ $? != 0 ];then
       echo "ERROR: the haproxy daemon is not running"
       cat /var/log/opsbro/generator.log
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi

    ls -thor /var/log
    ls -thor /var/log/haproxy
    cat /var/log/haproxy*
    cat /var/log/haproxy/*
    cat /var/log/messages

    ps axjf

    echo "HAPROXY: look if local proxying is valid  `date`"
    OUT=$(curl -s http://localhost)
    if [[ "$OUT" != "NODE-HTTP-1" ]] && [[ "$OUT" != "NODE-HTTP-2" ]]; then
         echo "HAPROXY `date` Cannot reach real HTTP servers from the local HAPROXY: $OUT"
         cat /var/log/opsbro/generator.log
         cat /etc/haproxy/haproxy.cfg
         exit 2
    fi

    ps axjf

    grep 'NODE-HTTP-2' /etc/haproxy/haproxy.cfg > /dev/null
    if [ $? != 0 ];then
       echo "ERROR: cannot find the NODE-HTTP-2 in the haproxy configuration"
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi
    echo "HAPROXY: `date` NODE-HTTP-2 is present"

    ps axjf

    grep 'NODE-HTTP-1' /etc/haproxy/haproxy.cfg > /dev/null
    if [ $? != 0 ];then
       echo "ERROR: cannot find the NODE-HTTP-1 in the haproxy configuration"
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi
    echo "HAPROXY: `date` NODE-HTTP-1 is present"

    echo "Haproxy stats"
    STATS=$(curl -s "http://admin:admin@localhost/stats;csv;norefresh")
    NB=$(echo "$STATS" | grep 'back_http' | grep -v BACKEND | wc -l)
    if [ "$NB" != "2" ];then
       echo "ERROR: Bad total number of back http in haproxy $NB"
       echo "$STATS"
       cat /var/log/opsbro/generator.log
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi

    ps axjf

    NB_UP=$(echo "$STATS" | grep 'back_http' | grep -v BACKEND | grep UP | wc -l)
    if [ "$NB" != "2" ];then
       echo "Bad number of back http in haproxy in UP $NB"
       echo "$STATS"
       exit 2
    fi


    printf "\n\n"
    echo "============================== [`date`] $CASE let the client know we are ready ================================="
    # Let the httproxy node know we are ready for checking the haproxy state
    opsbro gossip events add  "LB-READY-$CASE"

    printf "\n\n"
    echo "============================== [`date`] $CASE waiting for end ================================="

    opsbro gossip events add  "ENDING-$CASE"
    wait_step_event "ENDING"

    echo "$CASE node exiting  `date`"
    exit 0
fi


<commit_msg>Enh: TEST now the DEMO1 scenario is using event to synchronize between docker containers (some can be very slow to start)<commit_after>#!/usr/bin/env bash

CASE=$1



printf "\n\n"
echo "============================== [`date`] Configuration setup ================================="


function wait_step_event_node {
    # NOTE: we are using a large timeout because the haproxy node can be long to download & install haproxy
    opsbro gossip events wait "$1-$2" --timeout=180
    if [ $? != 0 ];then
       echo "ERROR: `date` the node $2 did not reach STEP $1"
       exit 2
    fi
    echo "NODE $2 did reach step $1 at `date`"
}

function wait_step_event {
    echo " * Waiting for other nodes events. Starts at `date`"
    wait_step_event_node $1 "NODE-HTTP-1"
    wait_step_event_node $1 "NODE-HTTP-2"
    wait_step_event_node $1 "NODE-HAPROXY"
    wait_step_event_node $1 "NODE-CLIENT"
}

echo "$CASE starts to run `date` $TRAVIS"


# Set a valid display name for debug
opsbro agent parameters set display_name "$CASE"


# Haproxy is the central node, it will ensure us that all nodes will talk togethers
if [ $CASE == "NODE-HAPROXY" ]; then
   opsbro agent parameters set proxy-node true
   opsbro agent parameters add groups demo_haproxy
fi


# Client node will need to expose nodes with DNS
if [ $CASE == "NODE-CLIENT" ]; then
   opsbro packs overload global.dns
   opsbro packs parameters set local.dns.port  53
   opsbro agent parameters add groups dns-listener
   printf "nameserver 127.0.0.1\n" > /etc/resolv.conf
fi


# We are ready, launch the whole daemons
printf "\n\n"
echo "============================== [`date`] Launching daemon ================================="
/etc/init.d/opsbro start

echo "Daemon started `date`"

# For debug purpose
ip addr show | grep 'scope global'




# Let the nodes join them selve.
# NOTE: the haproxy node will be slower to get here, because he need to install haproxy during the start
echo "Launching UDP detection `date`"
opsbro gossip detect --auto-join




printf "\n\n"
echo "============================== [`date`] STEP1 synchronization ================================="
echo "Adding my own event: STEP1-$CASE  at `date`"

opsbro gossip events add  "STEP1-$CASE"
wait_step_event "STEP1"

echo "All nodes are syncronized at `date`"





printf "\n\n"
echo "============================== [`date`] Checking we are 4 members ================================="
MEMBERS=$(opsbro gossip members)
NB_MEMBERS=$(echo "$MEMBERS" | grep 'docker-container' | wc -l)

printf "Current members: `date` $MEMBERS\n"


# Must have all 4 nodes available
if [ $NB_MEMBERS != 4 ]; then
   echo "`date` BAD number of members: $NB_MEMBERS"
   echo "$MEMBERS"
   cat /var/log/opsbro/gossip.log
   exit 2
fi


# Check that HTTP nodes are running well
if [ $CASE == "NODE-HTTP-1" ] || [ $CASE == "NODE-HTTP-2" ]; then

   printf "\n\n"
   echo "============================== [`date`] $CASE apache checking ================================="

   PAYLOAD=$(curl -s http://localhost/)
   echo "PAYLOAD $PAYLOAD"
   if [ "$PAYLOAD" != "$CASE" ]; then
      echo "ERROR: `date` the http page is not ready"
      exit 2
   fi

   # They must have the apache group
   opsbro detectors wait-group 'apache'
   if [ $? != 0 ];then
      echo "ERROR:  `date` the apache group was not detected"
      exit 2
   fi

   # Let the httproxy node know we are ready for checking the haproxy state
   opsbro gossip events add  "HTTP-READY-$CASE"

   echo "$CASE  `date` will wait for queries until all others nodes are done"

   opsbro gossip events add  "ENDING-$CASE"
   wait_step_event "ENDING"


   echo "$CASE exiting at `date`"
   exit 0
fi


# Haproxy compliance must have installed the haproxy package
if [ $CASE == "NODE-HAPROXY" ]; then

    printf "\n\n"
    echo "============================== [`date`] $CASE waiting for HTTP node to be ready ================================="

    ps axjf
    netstat -tulpn | grep 80

    # Wait until the http-1 and http-2 are ready to be queried
    wait_step_event_node "HTTP-READY" "NODE-HTTP-1"
    wait_step_event_node "HTTP-READY" "NODE-HTTP-2"


    printf "\n\n"
    echo "============================== [`date`] $CASE checking that haproxy is happy ================================="
    echo "HAPROXY: look to see if haproxy is installed  `date`"
    opsbro compliance wait-compliant "TEST HAPROXY" --timeout=60
    if [ $? != 0 ]; then
        echo "Haproxy compliance cannot be fixed in COMPLIANCE state in 60s"
        opsbro compliance state
        exit 2
    fi

    echo "Compliance history that did install HAPROXY   `date`"
    opsbro compliance state


    echo "The haproxy group should now be detected"
    opsbro detectors wait-group 'haproxy'
    if [ $? != 0 ];then
       echo "ERROR: the haproxy group was not detected"
       exit 2
    fi


    echo "Testing local proxying"
    /etc/init.d/haproxy status | grep 'haproxy is running'
    if [ $? != 0 ];then
       echo "ERROR: the haproxy daemon is not running"
       cat /var/log/opsbro/generator.log
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi

    ls -thor /var/log
    ls -thor /var/log/haproxy
    cat /var/log/haproxy*
    cat /var/log/haproxy/*
    cat /var/log/messages

    ps axjf
    netstat -tulpn | grep 80

    echo "HAPROXY: look if local proxying is valid  `date`"
    OUT=$(curl -s http://localhost)
    if [[ "$OUT" != "NODE-HTTP-1" ]] && [[ "$OUT" != "NODE-HTTP-2" ]]; then
         echo "HAPROXY `date` Cannot reach real HTTP servers from the local HAPROXY: $OUT"
         cat /var/log/opsbro/generator.log
         cat /etc/haproxy/haproxy.cfg
         exit 2
    fi

    ps axjf
    netstat -tulpn | grep 80

    grep 'NODE-HTTP-2' /etc/haproxy/haproxy.cfg > /dev/null
    if [ $? != 0 ];then
       echo "ERROR: cannot find the NODE-HTTP-2 in the haproxy configuration"
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi
    echo "HAPROXY: `date` NODE-HTTP-2 is present"

    ps axjf
    netstat -tulpn | grep 80

    grep 'NODE-HTTP-1' /etc/haproxy/haproxy.cfg > /dev/null
    if [ $? != 0 ];then
       echo "ERROR: cannot find the NODE-HTTP-1 in the haproxy configuration"
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi
    echo "HAPROXY: `date` NODE-HTTP-1 is present"

    echo "Haproxy stats"
    STATS=$(curl -s "http://admin:admin@localhost/stats;csv;norefresh")
    NB=$(echo "$STATS" | grep 'back_http' | grep -v BACKEND | wc -l)
    if [ "$NB" != "2" ];then
       echo "ERROR: Bad total number of back http in haproxy $NB"
       echo "$STATS"
       cat /var/log/opsbro/generator.log
       cat /etc/haproxy/haproxy.cfg
       exit 2
    fi

    ps axjf
    netstat -tulpn | grep 80

    NB_UP=$(echo "$STATS" | grep 'back_http' | grep -v BACKEND | grep UP | wc -l)
    if [ "$NB" != "2" ];then
       echo "Bad number of back http in haproxy in UP $NB"
       echo "$STATS"
       exit 2
    fi


    printf "\n\n"
    echo "============================== [`date`] $CASE let the client know we are ready ================================="
    # Let the httproxy node know we are ready for checking the haproxy state
    opsbro gossip events add  "LB-READY-$CASE"

    printf "\n\n"
    echo "============================== [`date`] $CASE waiting for end ================================="

    opsbro gossip events add  "ENDING-$CASE"
    wait_step_event "ENDING"

    echo "$CASE node exiting  `date`"
    exit 0
fi



{'fraction_non_alphanumeric': 0.178314737490588, 'fraction_numerical': 0.010609898008077212, 'mean_word_length': 2.98418325606763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4003522', 'n_tokens_mistral': 5063, 'n_tokens_neox': 4634, 'n_words': 1670}