<commit_before>#!/bin/sh
#
# configure ODA-OS ingestion engine
#
#======================================================================

. `dirname $0`/../lib_logging.sh
. `dirname $0`/../lib_apache.sh

info "Configuring Ingestion Engine ... "

#======================================================================
# NOTE: In ODA-OS, it is not expected to have mutiple instances of the Ingestion Engine

[ -z "$ODAOS_IEAS_HOME" ] && error "Missing the required ODAOS_IEAS_HOME variable!"
[ -z "$ODAOS_DM_HOME" ] && error "Missing the required ODAOS_DM_HOME variable!"
[ -z "$ODAOS_IE_HOME" ] && error "Missing the required ODAOS_IE_HOME variable!"
#[ -z "$ODAOSHOSTNAME" ] && error "Missing the required ODAOSHOSTNAME variable!"
[ -z "$ODAOSROOT" ] && error "Missing the required ODAOSROOT variable!"
[ -z "$ODAOSUSER" ] && error "Missing the required ODAOSUSER variable!"
[ -z "$ODAOSLOGDIR" ] && error "Missing the required ODAOSLOGDIR variable!"
#[ -z "$ODAOSTMPDIR" ] && error "Missing the required ODAOSTMPDIR variable!"
export ODAOSLOGDIR=${ODAOSLOGDIR:-/var/log/odaos}

#HOSTNAME="$ODAOSHOSTNAME"
INSTANCE="ingestion"
INSTROOT="$ODAOS_IE_HOME"
#
SETTINGS="${INSTROOT}/${INSTANCE}/settings.py"
#WSGI="${INSTROOT}/${INSTANCE}/${INSTANCE}/wsgi.py"
INGESTION_CONFIG="${INSTROOT}/ingestion_config.json"
MNGCMD="${INSTROOT}/manage.py"
#
IE_LOG="${ODAOSLOGDIR}/ingestion_engine.log"

service ingeng stop || :
#-------------------------------------------------------------------------------
# configuration

# ingestion_config.json - download manager
sudo -u "$ODAOSUSER" ex "$INGESTION_CONFIG" <<END
1,\$s:\("DownloadManagerDir"[	 ]*\:[	 ]*"\).*\("[	 ]*,\):\1$ODAOS_DM_HOME\2:
wq
END

# settings.py - scripts' directory
sudo -u "$ODAOSUSER" ex "$SETTINGS" <<END
1,\$s/\(^ALLOWED_HOSTS[	 ]*=[	 ]*\).*/\1['$HOSTNAME','127.0.0.1','::1']/
1,\$g/Following line is set by the ODA-OS installation script/d
1,\$s:^\(LOGGING_FILE[	 ]*=[	 ]*\).*$:\1"$IE_LOG":
/^\(LOGGING_FILE[	 ]*=[	 ]*\).*$/i
# Following line is set by the ODA-OS installation script. Do not edit!
.
1,\$s:^\(IE_SCRIPTS_DIR[	 ]*=[	 ]*\).*$:\1"$ODAOS_IEAS_HOME":
1,\$s:^\(IE_DEFAULT_INGEST_SCRIPT[	 ]*=[	 ]*\).*$:\1"product_ingest.sh":
1,\$s:^\(IE_DEFAULT_ADDPROD_SCRIPT[	 ]*=[	 ]*\).*$:\1"product_add.sh":
1,\$s:^\(IE_DEFAULT_DEL_SCRIPT[	 ]*=[	 ]*\).*$:\1"scenario_remove.sh":
1,\$s:^\(IE_DEFAULT_CATREG_SCRIPT[	 ]*=[	 ]*\).*$:\1"catalogue_register.sh":
1,\$s:^\(IE_DEFAULT_CATDEREG_SCRIPT[	 ]*=[	 ]*\).*$:\1"catalogue_deregister.sh":
1,\$s:^\(IE_S2ATM_PREPROCESS_SCRIPT[	 ]*=[	 ]*\).*$:\1"s2preprocessor.sh":
1,\$s:^\(IE_TAR_RESULT_SCRIPT[	 ]*=[	 ]*\).*$:\1"tar_result.sh":
1,\$s:^\(IE_DEFAULT_UQMD_SCRIPT[	 ]*=[	 ]*\).*$:\1"update_quality_metadata.sh":
wq
END

# set-up logging
sudo -u "$ODAOSUSER" ex "$SETTINGS" <<END
g/^DEBUG[	 ]*=/s#\(^DEBUG[	 ]*=[	 ]*\).*#\1False#
g/^LOGGING[	 ]*=/,/^}/d
i
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {<commit_msg>Turning on the debug mode.<commit_after>#!/bin/sh
#
# configure ODA-OS ingestion engine
#
#======================================================================

. `dirname $0`/../lib_logging.sh
. `dirname $0`/../lib_apache.sh

info "Configuring Ingestion Engine ... "

#======================================================================
# NOTE: In ODA-OS, it is not expected to have mutiple instances of the Ingestion Engine

[ -z "$ODAOS_IEAS_HOME" ] && error "Missing the required ODAOS_IEAS_HOME variable!"
[ -z "$ODAOS_DM_HOME" ] && error "Missing the required ODAOS_DM_HOME variable!"
[ -z "$ODAOS_IE_HOME" ] && error "Missing the required ODAOS_IE_HOME variable!"
#[ -z "$ODAOSHOSTNAME" ] && error "Missing the required ODAOSHOSTNAME variable!"
[ -z "$ODAOSROOT" ] && error "Missing the required ODAOSROOT variable!"
[ -z "$ODAOSUSER" ] && error "Missing the required ODAOSUSER variable!"
[ -z "$ODAOSLOGDIR" ] && error "Missing the required ODAOSLOGDIR variable!"
#[ -z "$ODAOSTMPDIR" ] && error "Missing the required ODAOSTMPDIR variable!"
export ODAOSLOGDIR=${ODAOSLOGDIR:-/var/log/odaos}

#HOSTNAME="$ODAOSHOSTNAME"
INSTANCE="ingestion"
INSTROOT="$ODAOS_IE_HOME"
#
SETTINGS="${INSTROOT}/${INSTANCE}/settings.py"
#WSGI="${INSTROOT}/${INSTANCE}/${INSTANCE}/wsgi.py"
INGESTION_CONFIG="${INSTROOT}/ingestion_config.json"
MNGCMD="${INSTROOT}/manage.py"
#
IE_LOG="${ODAOSLOGDIR}/ingestion_engine.log"

service ingeng stop || :
#-------------------------------------------------------------------------------
# configuration

# ingestion_config.json - download manager
sudo -u "$ODAOSUSER" ex "$INGESTION_CONFIG" <<END
1,\$s:\("DownloadManagerDir"[	 ]*\:[	 ]*"\).*\("[	 ]*,\):\1$ODAOS_DM_HOME\2:
wq
END

# settings.py - scripts' directory
sudo -u "$ODAOSUSER" ex "$SETTINGS" <<END
1,\$s/\(^ALLOWED_HOSTS[	 ]*=[	 ]*\).*/\1['$HOSTNAME','127.0.0.1','::1']/
1,\$g/Following line is set by the ODA-OS installation script/d
1,\$s:^\(LOGGING_FILE[	 ]*=[	 ]*\).*$:\1"$IE_LOG":
/^\(LOGGING_FILE[	 ]*=[	 ]*\).*$/i
# Following line is set by the ODA-OS installation script. Do not edit!
.
1,\$s:^\(IE_SCRIPTS_DIR[	 ]*=[	 ]*\).*$:\1"$ODAOS_IEAS_HOME":
1,\$s:^\(IE_DEFAULT_INGEST_SCRIPT[	 ]*=[	 ]*\).*$:\1"product_ingest.sh":
1,\$s:^\(IE_DEFAULT_ADDPROD_SCRIPT[	 ]*=[	 ]*\).*$:\1"product_add.sh":
1,\$s:^\(IE_DEFAULT_DEL_SCRIPT[	 ]*=[	 ]*\).*$:\1"scenario_remove.sh":
1,\$s:^\(IE_DEFAULT_CATREG_SCRIPT[	 ]*=[	 ]*\).*$:\1"catalogue_register.sh":
1,\$s:^\(IE_DEFAULT_CATDEREG_SCRIPT[	 ]*=[	 ]*\).*$:\1"catalogue_deregister.sh":
1,\$s:^\(IE_S2ATM_PREPROCESS_SCRIPT[	 ]*=[	 ]*\).*$:\1"s2preprocessor.sh":
1,\$s:^\(IE_TAR_RESULT_SCRIPT[	 ]*=[	 ]*\).*$:\1"tar_result.sh":
1,\$s:^\(IE_DEFAULT_UQMD_SCRIPT[	 ]*=[	 ]*\).*$:\1"update_quality_metadata.sh":
wq
END

# set-up logging
sudo -u "$ODAOSUSER" ex "$SETTINGS" <<END
g/^DEBUG[	 ]*=/s#\(^DEBUG[	 ]*=[	 ]*\).*#\1True#
g/^LOGGING[	 ]*=/,/^}/d
i
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'filters': {
        'require_debug_false': {
{'fraction_non_alphanumeric': 0.29098497495826375, 'fraction_numerical': 0.01302170283806344, 'mean_word_length': 4.397297297297297, 'pattern_counts': {'":': 20, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2994405', 'n_tokens_mistral': 2430, 'n_tokens_neox': 2391, 'n_words': 396}