<commit_before>HADOOP_HOME=/usr/lib/<%= node[:hadoop][:hadoop_handle] %>
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING-hadoop}

HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING-hdfs}
export HADOOP_HOME=/usr/lib/hadoop-0.20
export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
export HADOOP_DATANODE_USER=hdfs
export HADOOP_JOBTRACKER_USER=mapred
export HADOOP_TASKTRACKER_USER=mapred
export HADOOP_IDENT_STRING=hadoop

# The maximum amount of heap to use, in MB. Default is 1000.
export HADOOP_HEAPSIZE=<%= node[:hadoop][:hadoop_daemon_heapsize] %>

export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_namenode_heapsize]                            || node[:hadoop][:hadoop_daemon_heapsize] %>m"
export HADOOP_SECONDARYNAMENODE_OPTS="$HADOOP_SECONDARYNAMENODE_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_secondarynamenode_heapsize] || node[:hadoop][:hadoop_daemon_heapsize] %>m"
export HADOOP_JOBTRACKER_OPTS="$HADOOP_JOBTRACKER_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_jobtracker_heapsize]                      || node[:hadoop][:hadoop_daemon_heapsize] %>m"

<%- unless node[:hadoop][:extra_classpaths].nil? %>
# Extra classpaths for hadoop
export HADOOP_CLASSPATH="<%= node[:hadoop][:extra_classpaths].values.join(":") %>:$HADOOP_CLASSPATH"<commit_msg>Bind to IPv4 instead of IPv6 by default<commit_after>HADOOP_HOME=/usr/lib/<%= node[:hadoop][:hadoop_handle] %>
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING-hadoop}

HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING-hdfs}
export HADOOP_HOME=/usr/lib/hadoop-0.20
export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
export HADOOP_DATANODE_USER=hdfs
export HADOOP_JOBTRACKER_USER=mapred
export HADOOP_TASKTRACKER_USER=mapred
export HADOOP_IDENT_STRING=hadoop

# The maximum amount of heap to use, in MB. Default is 1000.
export HADOOP_HEAPSIZE=<%= node[:hadoop][:hadoop_daemon_heapsize] %>

# Ubuntu wants us to use IPv6. Hadoop doesn't support that, but nevertheless binds to :::50010. Let's tell it we don't agree.
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"

export HADOOP_NAMENODE_OPTS="$HADOOP_NAMENODE_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_namenode_heapsize]                            || node[:hadoop][:hadoop_daemon_heapsize] %>m"
export HADOOP_SECONDARYNAMENODE_OPTS="$HADOOP_SECONDARYNAMENODE_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_secondarynamenode_heapsize] || node[:hadoop][:hadoop_daemon_heapsize] %>m"
export HADOOP_JOBTRACKER_OPTS="$HADOOP_JOBTRACKER_OPTS -XX:+UseParallelGC -Xmx<%= node[:hadoop][:hadoop_jobtracker_heapsize]                      || node[:hadoop][:hadoop_daemon_heapsize] %>m"

<%- unless node[:hadoop][:extra_classpaths].nil? %>
# Extra classpaths for hadoop
export HADOOP_CLASSPATH="<%= node[:hadoop][:extra_classpaths].values.join(":") %>:$HADOOP_CLASSPATH"
{'fraction_non_alphanumeric': 0.12703357563170647, 'fraction_numerical': 0.007961232260297681, 'mean_word_length': 7.704819277108434, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6258467', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1118, 'n_words': 158}