<commit_before>#!/bin/bash

set -u
set -o errexit

cd `dirname $0`/../

echo "setting up test database"
./script/run_with_carton.sh ./script/mediawords_psql_wrapper.pl --dump-env-commands --db-label test > /tmp/test_db_$$
source /tmp/test_db_$$

if [[ "$PGDATABASE" != "mediacloud_test" ]]; then
    echo "aborting test database name ($PGDATABASE) must be mediacloud_test"
    exit -1
else
    RELOAD_TEST_DB=1

    if [ $RELOAD_TEST_DB = 1 ]; then
       echo "Dropping database $PGDATABASE"
       dropdb $PGDATABASE
       echo "Creating database $PGDATABASE"
       createdb $PGDATABASE
       echo "Loading test dump"
       psql --quiet -d $PGDATABASE -f data/db_dumps/cc_blogs_mc_db.sql
    fi
fi

UPGRADE_DB_SQL=`script/run_with_carton.sh script/mediawords_upgrade_db.pl --db_label test`

if [ ${#UPGRADE_DB_SQL} -gt 0 ]; then
    script/run_with_carton.sh script/mediawords_upgrade_db.pl --db_label test --import
    pg_dump --no-owner --no-acl --no-security-labels --format=plain mediacloud_test > data/db_dumps/cc_blogs_mc_db.sql
    echo "updated data/db_dumps/cc_blogs_mc_db.sql to new schema"
fi

echo "testing for runnning solr"

if ps aux | grep java | grep runSolr > /dev/null; then
    echo "Solr is already running (it shouldn't)."
    exit 1
fi

echo "Starting Solr..."
python ./solr/run_solr_standalone.py &
solr_pid=$!

SOLR_IS_UP=0
SOLR_START_RETRIES=90

echo "Waiting $SOLR_START_RETRIES seconds for Solr to start..."
for i in `seq 1 $SOLR_START_RETRIES`; do
    echo "Trying to connect (#$i)..."
    if nc -z -w 10 127.0.0.1 8983; then
        echo "Solr is up"
        SOLR_IS_UP=1
        break
    else
        # Still down
        sleep 1
    fi
done

if [ $SOLR_IS_UP = 1 ]; then
    echo "Solr is up."
else
    echo "Solr is down after $SOLR_START_RETRIES seconds, giving up."
    exit 1
fi

TEST_RETURN_STATUS=0

if [ -z ${MEDIACLOUD_ENABLE_PYTHON_API_TESTS+x} ]; then
    MEDIACLOUD_ENABLE_PYTHON_API_TESTS=0
fi

if MEDIACLOUD_IGNORE_DB_SCHEMA_VERSION=1 MEDIAWORDS_FORCE_USING_TEST_DATABASE=1 \
    ./script/run_with_carton.sh ./script/mediawords_import_solr_data.pl --delete; then

    echo "Importing Solr data succeeded."

    if MEDIACLOUD_IGNORE_DB_SCHEMA_VERSION=1 MEDIAWORDS_FORCE_USING_TEST_DATABASE=1 \<commit_msg>Try sleeping for 10 seconds before starting the API test<commit_after>#!/bin/bash

set -u
set -o errexit

cd `dirname $0`/../

echo "setting up test database"
./script/run_with_carton.sh ./script/mediawords_psql_wrapper.pl --dump-env-commands --db-label test > /tmp/test_db_$$
source /tmp/test_db_$$

if [[ "$PGDATABASE" != "mediacloud_test" ]]; then
    echo "aborting test database name ($PGDATABASE) must be mediacloud_test"
    exit -1
else
    RELOAD_TEST_DB=1

    if [ $RELOAD_TEST_DB = 1 ]; then
       echo "Dropping database $PGDATABASE"
       dropdb $PGDATABASE
       echo "Creating database $PGDATABASE"
       createdb $PGDATABASE
       echo "Loading test dump"
       psql --quiet -d $PGDATABASE -f data/db_dumps/cc_blogs_mc_db.sql
    fi
fi

UPGRADE_DB_SQL=`script/run_with_carton.sh script/mediawords_upgrade_db.pl --db_label test`

if [ ${#UPGRADE_DB_SQL} -gt 0 ]; then
    script/run_with_carton.sh script/mediawords_upgrade_db.pl --db_label test --import
    pg_dump --no-owner --no-acl --no-security-labels --format=plain mediacloud_test > data/db_dumps/cc_blogs_mc_db.sql
    echo "updated data/db_dumps/cc_blogs_mc_db.sql to new schema"
fi

echo "testing for runnning solr"

if ps aux | grep java | grep runSolr > /dev/null; then
    echo "Solr is already running (it shouldn't)."
    exit 1
fi

echo "Starting Solr..."
python ./solr/run_solr_standalone.py &
solr_pid=$!

SOLR_IS_UP=0
SOLR_START_RETRIES=90

echo "Waiting $SOLR_START_RETRIES seconds for Solr to start..."
for i in `seq 1 $SOLR_START_RETRIES`; do
    echo "Trying to connect (#$i)..."
    if nc -z -w 10 127.0.0.1 8983; then
        echo "Solr is up"
        SOLR_IS_UP=1
        break
    else
        # Still down
        sleep 1
    fi
done

if [ $SOLR_IS_UP = 1 ]; then
    echo "Solr is up."
else
    echo "Solr is down after $SOLR_START_RETRIES seconds, giving up."
    exit 1
fi

# Otherwise one might get "SolrCore is loading,code=503"
echo "Waiting for Solr to finish loading its cores..."
sleep 10

TEST_RETURN_STATUS=0

if [ -z ${MEDIACLOUD_ENABLE_PYTHON_API_TESTS+x} ]; then
    MEDIACLOUD_ENABLE_PYTHON_API_TESTS=0
fi

if MEDIACLOUD_IGNORE_DB_SCHEMA_VERSION=1 MEDIAWORDS_FORCE_USING_TEST_DATABASE=1 \
    ./script/run_with_carton.sh ./script/mediawords_import_solr_data.pl --delete; then

    echo "Importing Solr data succeeded."

    if MEDIACLOUD_IGNORE_DB_SCHEMA_VERSION=1 MEDIAWORDS_FORCE_USING_TEST_DATABASE=1 \
{'fraction_non_alphanumeric': 0.1013470173187941, 'fraction_numerical': 0.015180671370536669, 'mean_word_length': 3.351627906976744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1426058', 'n_tokens_mistral': 1909, 'n_tokens_neox': 1795, 'n_words': 449}