<commit_before>#!/bin/bash

# integration test for Socorro, using rabbitmq
#
# bring up components, submit test crash, ensure that it shows up in
# reports tables.
#
# This uses the same setup as http://socorro.readthedocs.org/en/latest/installation.html

if [ "$#" != "1" ] || [ "$1" != "--destroy" ]
then
  echo "WARNING - this script will destroy the local socorro install."
  echo "The default database and config files will be overwritten."
  echo "You must pass the --destroy flag to continue."
  exit 1
fi


if [ -z "$database_hostname" ]
then
  database_hostname="localhost"
fi

if [ -z "$database_username" ]
then
  database_username="breakpad_rw"
fi

if [ -z "$database_password" ]
then
  database_password="<PASSWORD>"
fi

if [ -z "$rmq_host" ]
then
  rmq_host="localhost"
fi

if [ -z "$RABBITMQ_USERNAME" ]
then
  RABBITMQ_USERNAME="guest"
fi

if [ -z "$RABBITMQ_PASSWORD" ]
then
  RABBITMQ_PASSWORD="<PASSWORD>"
fi

if [ -z "$RABBITMQ_VHOST" ]
then
  RABBITMQ_VHOST="/"
fi

function cleanup_rabbitmq() {
  echo -n "INFO: Purging rabbitmq queue 'socorro.normal'..."
  python scripts/test_rabbitmq.py --test_rabbitmq.purge='socorro.normal' --test_rabbitmq.rabbitmq_host=$rmq_host --test_rabbitmq.rabbitmq_user=$RABBITMQ_USERNAME --test_rabbitmq.rabbitmq_password=$RABBITMQ_PASSWORD --test_rabbitmq.rabbitmq_vhost=$RABBITMQ_VHOST > /dev/null 2>&1
  echo " Done."
  echo -n "INFO: Purging rabbitmq queue 'socorro.priority'..."
  python scripts/test_rabbitmq.py --test_rabbitmq.purge='socorro.priority' --test_rabbitmq.rabbitmq_host=$rmq_host --test_rabbitmq.rabbitmq_user=$RABBITMQ_USERNAME --test_rabbitmq.rabbitmq_password=$RABBIT<PASSWORD> --test_rabbitmq.rabbitmq_vhost=$RABBITMQ_VHOST > /dev/null 2>&1
  echo " Done."
}

function cleanup() {
  cleanup_rabbitmq

  echo "INFO: cleaning up crash storage directories"
  rm -rf ./primaryCrashStore/ ./processedCrashStore/
  rm -rf ./crashes/

  echo "INFO: Terminating background jobs"

  for p in collector processor middleware
  do
    # destroy any running processes started by this shell
    kill `jobs -p`
    # destroy anything trying to write to the log files too
    fuser -k ${p}.log > /dev/null 2>&1
  done

  return 0
}

trap 'cleanup' INT

function fatal() {
  exit_code=$1
  message=$2

  echo "ERROR: $message"

  cleanup

  exit $exit_code
}

echo -n "INFO: setting up environment..."
. socorro-virtualenv/bin/activate >> setup.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "could activate virtualenv"
fi
export PYTHONPATH=.
echo " Done."

echo -n "INFO: setting up database..."
python socorro/external/postgresql/setupdb_app.py --database_usernamename=$database_username --database_password=$database_password --database_name=breakpad --database_hostname=$database_hostname --dropdb --force --fakedata --fakedata_days=1 > setupdb.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "setupdb_app.py failed, check setupdb.log"
fi
echo " Done."
popd >> setupdb.log 2>&1

# ensure rabbitmq is really empty and no previous failure left garbage
cleanup_rabbitmq

echo -n "INFO: setting up 'weekly-reports-partitions' via crontabber..."
python socorro/cron/crontabber_app.py --resource.postgresql.database_hostname=$database_hostname --secrets.postgresql.database_usernamename=$database_username --secrets.postgresql.database_password=$database_password --job=weekly-reports-partitions --force >> setupdb.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "crontabber weekly-reports-partitions failed, check setupdb.log"
fi
echo " Done."

echo -n "INFO: configuring backend jobs..."
for p in collector processor middleware
do
  cp config/${p}.ini-dist config/${p}.ini
  if [ $? != 0 ]
  then
    fatal 1 "copying default config for $p failed"
  fi
  # ensure no running processes
  fuser -k ${p}.log > /dev/null 2>&1
done
echo " Done."

echo -n "INFO: starting up collector, processor and middleware..."
python socorro/collector/collector_app.py --admin.conf=./config/collector.ini --storage.storage1.host=$rmq_host --storage.storage1.rabbitmq_user=$RABBITMQ_USERNAME --storage.storage1.rabbitmq_password=$<PASSWORD> --storage.storage1.virtual_host=$RABBITMQ_VHOST --storage.storage1.transaction_executor_class=socorro.database.transaction_executor.TransactionExecutor --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy > collector.log 2>&1 &
python socorro/processor/processor_app.py --admin.conf=./config/processor.ini --processor.database_hostname=$database_hostname --new_crash_source.host=$rmq_host --new_crash_source.rabbitmq_user=$RABBITMQ_USERNAME --new_crash_source.rabbitmq_password=$<PASSWORD> --new_crash_source.virtual_host=$RABBITMQ_VHOST --destination.storage1.database_hostname=$database_hostname --registrar.database_hostname=$database_hostname > processor.log 2>&1 &
sleep 1
python socorro/middleware/middleware_app.py --admin.conf=./config/middleware.ini --database.database_hostname=$database_hostname --database.database_usernamename=$database_username --database.database_password=$database_password --rabbitmq.host=$rmq_host --rabbitmq.rabbitmq_user=$RABBITMQ_USERNAME --rabbitmq.rabbitmq_password=$RABBITMQ_PASSWORD --rabbitmq.virtual_host=$RABBITMQ_VHOST --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy > middleware.log 2>&1 &
echo " Done."

function retry() {
  name=$1
  search=$2
<commit_msg>Replace RABBITMQ_USERNAME envvar with rmq_user<commit_after>#!/bin/bash

# integration test for Socorro, using rabbitmq
#
# bring up components, submit test crash, ensure that it shows up in
# reports tables.
#
# This uses the same setup as http://socorro.readthedocs.org/en/latest/installation.html

if [ "$#" != "1" ] || [ "$1" != "--destroy" ]
then
  echo "WARNING - this script will destroy the local socorro install."
  echo "The default database and config files will be overwritten."
  echo "You must pass the --destroy flag to continue."
  exit 1
fi


if [ -z "$database_hostname" ]
then
  database_hostname="localhost"
fi

if [ -z "$database_username" ]
then
  database_username="breakpad_rw"
fi

if [ -z "$database_password" ]
then
  database_password="<PASSWORD>"
fi

if [ -z "$rmq_host" ]
then
  rmq_host="localhost"
fi

if [ -z "$rmq_user" ]
then
  rmq_user="guest"
fi

if [ -z "$RABBITMQ_PASSWORD" ]
then
  RABBITMQ_PASSWORD="<PASSWORD>"
fi

if [ -z "$RABBITMQ_VHOST" ]
then
  RABBITMQ_VHOST="/"
fi

function cleanup_rabbitmq() {
  echo -n "INFO: Purging rabbitmq queue 'socorro.normal'..."
  python scripts/test_rabbitmq.py --test_rabbitmq.purge='socorro.normal' --test_rabbitmq.rabbitmq_host=$rmq_host --test_rabbitmq.rabbitmq_user=$rmq_user --test_rabbitmq.rabbitmq_password=$RABBITMQ_PASSWORD --test_rabbitmq.rabbitmq_vhost=$RABBITMQ_VHOST > /dev/null 2>&1
  echo " Done."
  echo -n "INFO: Purging rabbitmq queue 'socorro.priority'..."
  python scripts/test_rabbitmq.py --test_rabbitmq.purge='socorro.priority' --test_rabbitmq.rabbitmq_host=$rmq_host --test_rabbitmq.rabbitmq_user=$rmq_user --test_rabbitmq.rabbitmq_password=<PASSWORD> --test_rabbitmq.rabbitmq_vhost=$RABBITMQ_VHOST > /dev/null 2>&1
  echo " Done."
}

function cleanup() {
  cleanup_rabbitmq

  echo "INFO: cleaning up crash storage directories"
  rm -rf ./primaryCrashStore/ ./processedCrashStore/
  rm -rf ./crashes/

  echo "INFO: Terminating background jobs"

  for p in collector processor middleware
  do
    # destroy any running processes started by this shell
    kill `jobs -p`
    # destroy anything trying to write to the log files too
    fuser -k ${p}.log > /dev/null 2>&1
  done

  return 0
}

trap 'cleanup' INT

function fatal() {
  exit_code=$1
  message=$2

  echo "ERROR: $message"

  cleanup

  exit $exit_code
}

echo -n "INFO: setting up environment..."
. socorro-virtualenv/bin/activate >> setup.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "could activate virtualenv"
fi
export PYTHONPATH=.
echo " Done."

echo -n "INFO: setting up database..."
python socorro/external/postgresql/setupdb_app.py --database_usernamename=$database_username --database_password=$database_password --database_name=breakpad --database_hostname=$database_hostname --dropdb --force --fakedata --fakedata_days=1 > setupdb.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "setupdb_app.py failed, check setupdb.log"
fi
echo " Done."
popd >> setupdb.log 2>&1

# ensure rabbitmq is really empty and no previous failure left garbage
cleanup_rabbitmq

echo -n "INFO: setting up 'weekly-reports-partitions' via crontabber..."
python socorro/cron/crontabber_app.py --resource.postgresql.database_hostname=$database_hostname --secrets.postgresql.database_usernamename=$database_username --secrets.postgresql.database_password=$database_password --job=weekly-reports-partitions --force >> setupdb.log 2>&1
if [ $? != 0 ]
then
  fatal 1 "crontabber weekly-reports-partitions failed, check setupdb.log"
fi
echo " Done."

echo -n "INFO: configuring backend jobs..."
for p in collector processor middleware
do
  cp config/${p}.ini-dist config/${p}.ini
  if [ $? != 0 ]
  then
    fatal 1 "copying default config for $p failed"
  fi
  # ensure no running processes
  fuser -k ${p}.log > /dev/null 2>&1
done
echo " Done."

echo -n "INFO: starting up collector, processor and middleware..."
python socorro/collector/collector_app.py --admin.conf=./config/collector.ini --storage.storage1.host=$rmq_host --storage.storage1.rabbitmq_user=$rmq_user --storage.storage1.rabbitmq_password=$<PASSWORD> --storage.storage1.virtual_host=$RABBITMQ_VHOST --storage.storage1.transaction_executor_class=socorro.database.transaction_executor.TransactionExecutor --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy > collector.log 2>&1 &
python socorro/processor/processor_app.py --admin.conf=./config/processor.ini --processor.database_hostname=$database_hostname --new_crash_source.host=$rmq_host --new_crash_source.rabbitmq_user=$rmq_user --new_crash_source.rabbitmq_password=$<PASSWORD> --new_crash_source.virtual_host=$RABBITMQ_VHOST --destination.storage1.database_hostname=$database_hostname --registrar.database_hostname=$database_hostname > processor.log 2>&1 &
sleep 1
python socorro/middleware/middleware_app.py --admin.conf=./config/middleware.ini --database.database_hostname=$database_hostname --database.database_usernamename=$database_username --database.database_password=$database_password --rabbitmq.host=$rmq_host --rabbitmq.rabbitmq_user=$rmq_user --rabbitmq.rabbitmq_password=$<PASSWORD> --rabbitmq.virtual_host=$RABBITMQ_VHOST --web_server.wsgi_server_class=socorro.webapi.servers.CherryPy > middleware.log 2>&1 &
echo " Done."

function retry() {
  name=$1
  search=$2

{'fraction_non_alphanumeric': 0.12457240592930445, 'fraction_numerical': 0.008741923223109084, 'mean_word_length': 4.748225013653741, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4315620', 'n_tokens_mistral': 3719, 'n_tokens_neox': 3497, 'n_words': 816}