<commit_before>#!/bin/bash
# *
# * This Source Code Form is subject to the terms of the Mozilla Public
# * License, v. 2.0. If a copy of the MPL was not distributed with this
# * file, You can obtain one at http://mozilla.org/MPL/2.0/.
# *

USAGE="Usage: $0 [--production] [--nostart] [--keep_node_modules] [--continue_on_db_error]"

production=no
dostart=yes
keep_node_modules=no
continue_on_db_error=no

# Run through all flags and match
while test $# -gt 0; do
	case "$1" in
		--production)
			shift
			production=yes
			;;
		--nostart)
			shift
			dostart=no
			;;
		--keep_node_modules)
			shift
			keep_node_modules=yes
			;;
		--continue_on_db_error)
			shift
			continue_on_db_error=yes
			;;
		"")
			# Empty string so just ignore. Esp. happens if no install_args on docker-compose up.
			shift
			;;
		*)
			printf "Unknown argument \"%s\"\n" $1
			printf "%s\n" $USAGE
			exit 1
	esac
done

# Load .env if it exists

if [ -f ".env" ]; then
	source .env
fi

# Install NPM dependencies
if [ "$keep_node_modules" == "yes" ]; then
	printf "%s\n" "skipping NPM install as requested"
else
	printf "%s\n" "NPM install..."
	npm ci --loglevel=warn
	if [ $? == 0 ]; then
		printf "%s\n" "NPM install finished."
	else
		# npm reported an error. Sometimes it does so can skip steps.
		# Using printf since it is more reliable.
		printf "\n%s\n" "NPM reported an error so stopping"
		exit 2
	fi
fi

create_error=0 # Boolean

tries=0
max_tries=10

# Try to create the schema until it succeeds
while [ $create_error == 0 ]; do
	# See if exceed allowed number of tries and stop install if you have.
	if [ $tries -ge $max_tries ]; then
		printf "%s\n" "FAILED! Too many tries. Try again but then check if your database at $OED_DB_HOST:$OED_DB_PORT is down."
		exit 1
	fi
    # Sleep to let PostgreSQL chill out
    sleep 5
    printf "%s\n" "Attempting to create database."
    # Redirect stderr to a file
    npm run createdb |& tee /tmp/oed.error > /dev/null
	createdb_code=${PIPESTATUS[0]}
	if [ $createdb_code -ne 0 ]; then
		# An error occurred during createdb.
		# search the file for the kind of error we can recover from<commit_msg>reset sleep to see if helps with Travis failure<commit_after>#!/bin/bash
# *
# * This Source Code Form is subject to the terms of the Mozilla Public
# * License, v. 2.0. If a copy of the MPL was not distributed with this
# * file, You can obtain one at http://mozilla.org/MPL/2.0/.
# *

USAGE="Usage: $0 [--production] [--nostart] [--keep_node_modules] [--continue_on_db_error]"

production=no
dostart=yes
keep_node_modules=no
continue_on_db_error=no

# Run through all flags and match
while test $# -gt 0; do
	case "$1" in
		--production)
			shift
			production=yes
			;;
		--nostart)
			shift
			dostart=no
			;;
		--keep_node_modules)
			shift
			keep_node_modules=yes
			;;
		--continue_on_db_error)
			shift
			continue_on_db_error=yes
			;;
		"")
			# Empty string so just ignore. Esp. happens if no install_args on docker-compose up.
			shift
			;;
		*)
			printf "Unknown argument \"%s\"\n" $1
			printf "%s\n" $USAGE
			exit 1
	esac
done

# Load .env if it exists

if [ -f ".env" ]; then
	source .env
fi

# Install NPM dependencies
if [ "$keep_node_modules" == "yes" ]; then
	printf "%s\n" "skipping NPM install as requested"
else
	printf "%s\n" "NPM install..."
	npm ci --loglevel=warn
	if [ $? == 0 ]; then
		printf "%s\n" "NPM install finished."
	else
		# npm reported an error. Sometimes it does so can skip steps.
		# Using printf since it is more reliable.
		printf "\n%s\n" "NPM reported an error so stopping"
		exit 2
	fi
fi

create_error=0 # Boolean

tries=0
max_tries=10

# Try to create the schema until it succeeds
while [ $create_error == 0 ]; do
	# See if exceed allowed number of tries and stop install if you have.
	if [ $tries -ge $max_tries ]; then
		printf "%s\n" "FAILED! Too many tries. Try again but then check if your database at $OED_DB_HOST:$OED_DB_PORT is down."
		exit 1
	fi
    # Sleep to let PostgreSQL chill out
    sleep 1
    printf "%s\n" "Attempting to create database."
    # Redirect stderr to a file
    npm run createdb |& tee /tmp/oed.error > /dev/null
	createdb_code=${PIPESTATUS[0]}
	if [ $createdb_code -ne 0 ]; then
		# An error occurred during createdb.
		# search the file for the kind of error we can recover from
{'fraction_non_alphanumeric': 0.11204091120409113, 'fraction_numerical': 0.009298000929800094, 'mean_word_length': 3.1177033492822965, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7429139', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1494, 'n_words': 553}