<commit_before>#!/bin/sh
set -x -e

DCI_ES_DIR=${DCI_ES_DIR:-".es_dir"}

# get dci_es_dir absolute path
DCI_ES_DIR="$(cd $(dirname "$0") && pwd)/$DCI_ES_DIR"

# if elasticsearch is already running we do not want to run this script
[ ! -z "$DISABLE_ES_START" ] &&exit 0

# checks if not already running
PROCESS=$(ps auxfw | grep elasticsearch | grep -v grep | awk '{print $2}')
echo $PROCESS
if [ "$PROCESS" != "" ]; then kill $PROCESS; fi

[ -d "$DCI_ES_DIR" ] && rm -rf "$DCI_ES_DIR"

# init the es directory and start the process
mkdir -p ${DCI_ES_DIR}/config ${DCI_ES_DIR}/logs ${DCI_ES_DIR}/data
cp -r /usr/share/elasticsearch/* $DCI_ES_DIR/
echo "network.host: 0.0.0.0" > ${DCI_ES_DIR}/config/elasticsearch.yml
${DCI_ES_DIR}/bin/elasticsearch -d
until $(netstat -lntp | grep -q ":9200 "); do
    sleep 1
done<commit_msg>[start_es]: fix start_es.sh race condition<commit_after>#!/bin/sh
set -x

DCI_ES_DIR=${DCI_ES_DIR:-".es_dir"}

# get dci_es_dir absolute path
DCI_ES_DIR="$(cd $(dirname "$0") && pwd)/$DCI_ES_DIR"

# if elasticsearch is already running we do not want to run this script
[ ! -z "$DISABLE_ES_START" ] &&exit 0

# checks if not already running
PROCESS=$(ps auxfw | grep elasticsearch | grep -v grep | awk '{print $2}')
echo $PROCESS
if [ "$PROCESS" != "" ]; then kill $PROCESS; fi

[ -d "$DCI_ES_DIR" ] && rm -rf "$DCI_ES_DIR"

# init the es directory and start the process
mkdir -p ${DCI_ES_DIR}/config ${DCI_ES_DIR}/logs ${DCI_ES_DIR}/data
cp -r /usr/share/elasticsearch/* $DCI_ES_DIR/
echo "network.host: 0.0.0.0" > ${DCI_ES_DIR}/config/elasticsearch.yml
${DCI_ES_DIR}/bin/elasticsearch -d
while true; do
    status=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9200/_cluster/health)
    if [ ${status} -eq 200 ]; then
        break
    fi
    sleep 1
done

{'fraction_non_alphanumeric': 0.16508644729503624, 'fraction_numerical': 0.018404907975460124, 'mean_word_length': 4.067796610169491, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7087873', 'n_tokens_mistral': 724, 'n_tokens_neox': 699, 'n_words': 194}