<commit_before>#!/bin/bash

set -e
set -u

DIR="$(dirname $0)"

dc() {
	docker-compose -p hr -f ${DIR}/docker-compose.yml $*;
}

trap 'dc kill ; dc rm -f' EXIT

echo "Do we have OS password?"
echo $OS_PASSWORD

rm -rf ${DIR}/backups
mkdir -p ${DIR}/backups

dc build --pull

dc up -d database

dc run --rm tests

# load latest bag into database
echo "load latest bag database"
dc exec -T database update-db.sh atlas

echo "create hr api database / reset elastic index"
# create the hr_data and reset elastic
dc run --rm importer

echo "DONE! importing mks into database"

echo "create hr dump"
# run the backup shizzle
dc run --rm db-backup

echo "create hr index"
dc up importer_el1 importer_el2 importer_el3

# wait until all building is done
import_status=`docker wait hr_importer_el1_1 hr_importer_el2_1 hr_importer_el3_1`

# count the errors.
import_error=`echo $import_status | grep -o "1" | wc -l`

echo $import_error

if [ $import_error > 0 ]
then
    echo 'Elastic Import Error. 1 or more workers failed'
    exit -1
fi

dc run --rm el-backup

echo "DONE! with everything! You are awesome! <3"<commit_msg>bug fix fail import -1 is invalid<commit_after>#!/bin/bash

set -e
set -u

DIR="$(dirname $0)"

dc() {
	docker-compose -p hr -f ${DIR}/docker-compose.yml $*;
}

trap 'dc kill ; dc rm -f' EXIT

echo "Do we have OS password?"
echo $OS_PASSWORD

rm -rf ${DIR}/backups
mkdir -p ${DIR}/backups

dc build --pull

dc up -d database

dc run --rm tests

# load latest bag into database
echo "load latest bag database"
dc exec -T database update-db.sh atlas

echo "create hr api database / reset elastic index"
# create the hr_data and reset elastic
dc run --rm importer

echo "DONE! importing mks into database"

echo "create hr dump"
# run the backup shizzle
dc run --rm db-backup

echo "create hr index"
dc up importer_el1 importer_el2 importer_el3

# wait until all building is done
import_status=`docker wait hr_importer_el1_1 hr_importer_el2_1 hr_importer_el3_1`

# count the errors.
import_error=`echo $import_status | grep -o "1" | wc -l`

echo $import_error

if [ $import_error > 0 ]
then
    echo 'Elastic Import Error. 1 or more workers failed'
    exit 1
fi

dc run --rm el-backup

echo "DONE! with everything! You are awesome! <3"
{'fraction_non_alphanumeric': 0.09968847352024922, 'fraction_numerical': 0.013796172674677348, 'mean_word_length': 3.6735966735966734, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761869', 'n_tokens_mistral': 828, 'n_tokens_neox': 794, 'n_words': 319}