<commit_before>#!/bin/bash -e
#
#
# Set Datafari environment variable
#
#

export DATAFARI_USER="datafari"
export POSTGRES_USER="postgres"

export DATAFARI_HOME=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PATH=${PATH}:$JAVA_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export TRUSTSTORE_PATH=${DATAFARI_HOME}/ssl-keystore/datafari-truststore.p12
export TRUSTSTORE_PASSWORD=<PASSWORD>
export JAVA_OPTS="${JAVA_OPTS} -Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD} -Duser.timezone=UTC"
export INIT_STATE_FILE=${DATAFARI_HOME}/bin/common/init_state.properties
export CONFIG_FILE=${DATAFARI_HOME}/tomcat/conf/datafari.properties
export SOLR_CONFIG_FILE=${DATAFARI_HOME}/tomcat/conf/solr.properties
export LOG4J_VERSION=2.13.3
export LD_LIBRARY_PATH=${DATAFARI_HOME}/pgsql/lib
export TMP_DIR=/tmp


# Home dirs
export PID_DIR=$DATAFARI_HOME/pid
export TOMCAT_HOME=$DATAFARI_HOME/tomcat
export TOMCAT_MCF_HOME=$DATAFARI_HOME/tomcat-mcf
export MCF_HOME=$DATAFARI_HOME/mcf/mcf_home
export SOLR_INSTALL_DIR=$DATAFARI_HOME/solr
export SOLR_ENV=$SOLR_INSTALL_DIR/bin/solr.in.sh
export SOLR_INCLUDE=$SOLR_ENV
export CASSANDRA_HOME=$DATAFARI_HOME/cassandra
export CASSANDRA_ENV=$CASSANDRA_HOME/bin/cassandra.in.sh
export CASSANDRA_INCLUDE=$CASSANDRA_ENV
export CASSANDRA_HOST=127.0.0.1
export CASSANDRA_PORT=9042
export POSTGRES_HOME=$DATAFARI_HOME/pgsql
export ZK_HOME=$DATAFARI_HOME/zookeeper
export ZK_ENV=$ZK_HOME/bin/zkEnvh.sh
export ZK_HOME_MCF=$DATAFARI_HOME/zookeeper-mcf
export ZK_ENV_MCF=$ZK_HOME_MCF/bin/zkEnvh.sh
export ELK_HOME=$DATAFARI_HOME/elk
export TIKA_SERVER_HOME=$DATAFARI_HOME/tika-server

# Logs directory: needed for Cassandra startup
export DATAFARI_LOGS=$DATAFARI_HOME/logs

# Configs
export CONFIG_HOME=$TOMCAT_HOME/conf
export TOMCAT_MCF_CONFIG_HOME=$TOMCAT_MCF_HOME/conf
export MAIN_DATAFARI_CONFIG_HOME=$TOMCAT_HOME/conf
export DATAFARI_SOLR_PROPERTIES_HOME=$TOMCAT_HOME/conf


# pid files 
export MCF_PID_FILE=$PID_DIR/mcf_crawler_agent.pid
export CATALINA_PID=$PID_DIR/tomcat.pid
export CATALINA_MCF_PID=$PID_DIR/tomcat-mcf.pid
export SOLR_PID_FILE=$PID_DIR/solr-8983.pid
export CASSANDRA_PID_FILE=$PID_DIR/cassandra.pid
export POSTGRES_PID_FILE=$PID_DIR/postmaster.pid
export ZK_PID_FILE=$PID_DIR/zookeeper-solr.pid
export ZK_MCF_PID_FILE=$PID_DIR/zookeeper-mcf.pid
if [ -d /etc/apache2 ]; then
	export APACHE_PID_FILE=$PID_DIR/apache2.pid
elif [ -d /etc/httpd ]; then
	export APACHE_PID_FILE=$PID_DIR/apache/apache2.pid
fi

export RETRIES_NUMBER=35<commit_msg>Update log4j version into datafari env<commit_after>#!/bin/bash -e
#
#
# Set Datafari environment variable
#
#

export DATAFARI_USER="datafari"
export POSTGRES_USER="postgres"

export DATAFARI_HOME=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )
export PATH=${PATH}:$JAVA_HOME/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
export TRUSTSTORE_PATH=${DATAFARI_HOME}/ssl-keystore/datafari-truststore.p12
export TRUSTSTORE_PASSWORD=<PASSWORD>
export JAVA_OPTS="${JAVA_OPTS} -Djavax.net.ssl.trustStore=${TRUSTSTORE_PATH} -Djavax.net.ssl.trustStorePassword=${TRUSTSTORE_PASSWORD} -Duser.timezone=UTC"
export INIT_STATE_FILE=${DATAFARI_HOME}/bin/common/init_state.properties
export CONFIG_FILE=${DATAFARI_HOME}/tomcat/conf/datafari.properties
export SOLR_CONFIG_FILE=${DATAFARI_HOME}/tomcat/conf/solr.properties
export LOG4J_VERSION=2.17.1
export LD_LIBRARY_PATH=${DATAFARI_HOME}/pgsql/lib
export TMP_DIR=/tmp


# Home dirs
export PID_DIR=$DATAFARI_HOME/pid
export TOMCAT_HOME=$DATAFARI_HOME/tomcat
export TOMCAT_MCF_HOME=$DATAFARI_HOME/tomcat-mcf
export MCF_HOME=$DATAFARI_HOME/mcf/mcf_home
export SOLR_INSTALL_DIR=$DATAFARI_HOME/solr
export SOLR_ENV=$SOLR_INSTALL_DIR/bin/solr.in.sh
export SOLR_INCLUDE=$SOLR_ENV
export CASSANDRA_HOME=$DATAFARI_HOME/cassandra
export CASSANDRA_ENV=$CASSANDRA_HOME/bin/cassandra.in.sh
export CASSANDRA_INCLUDE=$CASSANDRA_ENV
export CASSANDRA_HOST=127.0.0.1
export CASSANDRA_PORT=9042
export POSTGRES_HOME=$DATAFARI_HOME/pgsql
export ZK_HOME=$DATAFARI_HOME/zookeeper
export ZK_ENV=$ZK_HOME/bin/zkEnvh.sh
export ZK_HOME_MCF=$DATAFARI_HOME/zookeeper-mcf
export ZK_ENV_MCF=$ZK_HOME_MCF/bin/zkEnvh.sh
export ELK_HOME=$DATAFARI_HOME/elk
export TIKA_SERVER_HOME=$DATAFARI_HOME/tika-server

# Logs directory: needed for Cassandra startup
export DATAFARI_LOGS=$DATAFARI_HOME/logs

# Configs
export CONFIG_HOME=$TOMCAT_HOME/conf
export TOMCAT_MCF_CONFIG_HOME=$TOMCAT_MCF_HOME/conf
export MAIN_DATAFARI_CONFIG_HOME=$TOMCAT_HOME/conf
export DATAFARI_SOLR_PROPERTIES_HOME=$TOMCAT_HOME/conf


# pid files 
export MCF_PID_FILE=$PID_DIR/mcf_crawler_agent.pid
export CATALINA_PID=$PID_DIR/tomcat.pid
export CATALINA_MCF_PID=$PID_DIR/tomcat-mcf.pid
export SOLR_PID_FILE=$PID_DIR/solr-8983.pid
export CASSANDRA_PID_FILE=$PID_DIR/cassandra.pid
export POSTGRES_PID_FILE=$PID_DIR/postmaster.pid
export ZK_PID_FILE=$PID_DIR/zookeeper-solr.pid
export ZK_MCF_PID_FILE=$PID_DIR/zookeeper-mcf.pid
if [ -d /etc/apache2 ]; then
	export APACHE_PID_FILE=$PID_DIR/apache2.pid
elif [ -d /etc/httpd ]; then
	export APACHE_PID_FILE=$PID_DIR/apache/apache2.pid
fi

export RETRIES_NUMBER=35
{'fraction_non_alphanumeric': 0.10924531943640224, 'fraction_numerical': 0.010615711252653927, 'mean_word_length': 8.28673835125448, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5369627', 'n_tokens_mistral': 2306, 'n_tokens_neox': 2155, 'n_words': 262}