<commit_before>#
# Configuration for aidr-persister
#
#############################
# MANDATORY configuration
# Values must be set at each installation
#############################

#default location to store tweets - User must modify as per local file path
DEFAULT_PERSISTER_FILE_PATH=${DEFAULT_PERSISTER_FILE_PATH}
#Data location of the deployed persister app
#User must enter URL for accessing stored tweets 
SCD1_URL=http://${PERSISTER_HOST}:${PERSISTER_PORT}/${PERSISTER_WEB_APP_CONTEXT}/data/persister/

#############################
# OPTIONAL configuration
# Values can be left at their default values
#############################

FETCHER_CHANNEL=${COLLECTOR_CHANNEL_BASENAME}.
TAGGER_CHANNEL=${TAGGER_CHANNEL_BASENAME}.
COLLECTION_CHANNEL=${COLLECTION_CHANNEL_BASENAME}.

DEFAULT_PERSISTER=default_persister
DEFAULT_PERSISTER_CODE=default_code

#100K limit imposed by Twitter
DEFAULT_FILE_VOLUMN_LIMIT=100000
TWEETS_EXPORT_LIMIT_100K=100000

#10485760 #10MB buffer size
DEFAULT_FILE_WRITER_BUFFER_SIZE=1024
#max. number of tweets downloadable from AIDR website
DEFAULT_TWEETID_VOLUME_LIMIT=1000000

#errorlog.txt
LOG_FILE_NAME=AIDR_Persister_Log.txt

# Related to Tweet Download limit
TWEET_DOWNLOAD_LIMIT_MSG_PREFIX=The size of the collection is larger than
TWEET_DOWNLOAD_LIMIT_MSG_SUFFIX=. To download the full tweet IDs collection, please contact the AIDR admin at <EMAIL>
PERSISTER_CHANGE_NOTIFY_MSG=The directory structure of the AIDR Persistence module has changed. To download your old files, please contact the AIDR admin at <EMAIL>

#############################
# INHERITED configuration
# Values are inherited from common configuration
#############################

managerUrl=http://${MANAGER_HOST}:${MANAGER_PORT}/${MANAGER_WEB_APP_CONTEXT}
PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT}
PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES}
INPUT_PORT=${INPUT_PORT}
OUTPUT_PORT=${OUTPUT_PORT}
REDIS_HOST=${REDIS_HOST}
REDIS_PORT=${REDIS_PORT}<commit_msg>added STATUS_CODE_ERROR ptoperty to the property file in the persister module.<commit_after>#
# Configuration for aidr-persister
#
#############################
# MANDATORY configuration
# Values must be set at each installation
#############################

#default location to store tweets - User must modify as per local file path
DEFAULT_PERSISTER_FILE_PATH=${DEFAULT_PERSISTER_FILE_PATH}
#Data location of the deployed persister app
#User must enter URL for accessing stored tweets 
SCD1_URL=http://${PERSISTER_HOST}:${PERSISTER_PORT}/${PERSISTER_WEB_APP_CONTEXT}/data/persister/

#############################
# OPTIONAL configuration
# Values can be left at their default values
#############################

FETCHER_CHANNEL=${COLLECTOR_CHANNEL_BASENAME}.
TAGGER_CHANNEL=${TAGGER_CHANNEL_BASENAME}.
COLLECTION_CHANNEL=${COLLECTION_CHANNEL_BASENAME}.

DEFAULT_PERSISTER=default_persister
DEFAULT_PERSISTER_CODE=default_code

#100K limit imposed by Twitter
DEFAULT_FILE_VOLUMN_LIMIT=100000
TWEETS_EXPORT_LIMIT_100K=100000

#10485760 #10MB buffer size
DEFAULT_FILE_WRITER_BUFFER_SIZE=1024
#max. number of tweets downloadable from AIDR website
DEFAULT_TWEETID_VOLUME_LIMIT=1000000

#errorlog.txt
LOG_FILE_NAME=AIDR_Persister_Log.txt

# Related to Tweet Download limit
TWEET_DOWNLOAD_LIMIT_MSG_PREFIX=The size of the collection is larger than
TWEET_DOWNLOAD_LIMIT_MSG_SUFFIX=. To download the full tweet IDs collection, please contact the AIDR admin at <EMAIL>
PERSISTER_CHANGE_NOTIFY_MSG=The directory structure of the AIDR Persistence module has changed. To download your old files, please contact the AIDR admin at <EMAIL>

#############################
# INHERITED configuration
# Values are inherited from common configuration
#############################

managerUrl=http://${MANAGER_HOST}:${MANAGER_PORT}/${MANAGER_WEB_APP_CONTEXT}
PERSISTER_LOAD_LIMIT=${PERSISTER_LOAD_LIMIT}
PERSISTER_LOAD_CHECK_INTERVAL_MINUTES=${PERSISTER_LOAD_CHECK_INTERVAL_MINUTES}
INPUT_PORT=${INPUT_PORT}
OUTPUT_PORT=${OUTPUT_PORT}
REDIS_HOST=${REDIS_HOST}
REDIS_PORT=${REDIS_PORT}
STATUS_CODE_ERROR=${STATUS_CODE_ERROR}
{'fraction_non_alphanumeric': 0.1441419543023821, 'fraction_numerical': 0.019445794846864366, 'mean_word_length': 9.417721518987342, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1364992', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1262, 'n_words': 302}