<commit_before># Sample configuration file
export HADOOP_NAMENODE="namenode:8020"
export HADOOP_USER_NAME=username
<commit_msg>Allows override of defaults in config file<commit_after># Sample configuration file
export HADOOP_NAMENODE="namenode:8020"
export HADOOP_USER_NAME=username

# Override default configuration
# export BACKUP_ROOT="/backups"
# export BACKUP_HOSTDIR="${BACKUP_ROOT}/$(hostname -f | sed -E 'H;g;:t;s/(.*)(\n)(.*)(\.)(.*)/\1\4\5\2\3/;tt;s/\.(.*)\n/\1./')"
# export BACKUP_DIR="${BACKUP_HOSTDIR}/$(date +%Y%m%d%H%M)"
# export CONFIG_DIR="/etc/hdfsbackup"

{'fraction_non_alphanumeric': 0.18782608695652173, 'fraction_numerical': 0.02434782608695652, 'mean_word_length': 6.578947368421052, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7273743', 'n_tokens_mistral': 232, 'n_tokens_neox': 217, 'n_words': 38}