<commit_before><?xml version="1.0"?>
<!--
################################################
#
#              Generated by Chef
#
################################################
-->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value><%= node['bcpc']['hadoop']['hdfs_url']%></value>
  </property>

  <property>
    <name>hadoop.proxyuser.httpfs.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.httpfs.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hue.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hue.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.oozie.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.oozie.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>io.compression.codecs</name>
    <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
  </property>

  <property>
    <name>mapreduce.cluster.local.dir</name>
    <value><%=@mounts.map{ |d| "file:///disk/#{d}/yarn/mapred-local" }.join(",")%></value>
  </property>

  <property>
     <name>yarn.nodemanager.log-dirs</name>
    <value><%=@mounts.map{ |d| "file:///disk/#{d}/yarn/logs" }.join(",")%></value>
  </property>

  <property>
    <name>yarn.nodemanager.linux-container-executor.group</name>
    <value>yarn</value>
  </property>

  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>

  <property>
    <name>net.topology.script.file.name</name>
    <value>/etc/hadoop/conf/topology</value>
  </property>

  <property>
    <name>io.compression.codec.lzo.class</name>
    <value>com.hadoop.compression.lzo.LzoCodec</value>
  </property>

  <% if node[:bcpc][:hadoop][:kerberos][:enable] == true then %>
  <property>
    <name>hadoop.security.authentication</name>
    <value>kerberos</value>
    <description>Set theauthentication for the cluster. Valid values are: simple or kerberos.</description>
  </property>

  <property>
    <name>hadoop.security.authorization</name>
    <value>true</value>
    <description>Enable authorization for different protocols.</description>
  </property>

  <property>
    <name>hadoop.security.auth_to_local</name>
    <value>
      <% node[:bcpc][:hadoop][:kerberos][:data].each do|ke,va| %>
      RULE:[2:$1@$0](<%= va['principal']%>@.*<%= node[:bcpc][:hadoop][:kerberos][:realm] %>)s/.*/<%=va['owner'] %>/
      <% end %>
      DEFAULT
    </value>
    <description>
      The mapping from kerberos principal names to local OS user names.
    </description>
  </property>
  <% end %>

  <property>
    <name>hadoop.proxyuser.hive.hosts</name>
    <value>*</value>
  </property>
  
  <property>
    <name>hadoop.proxyuser.hive.groups</name>
    <value>*</value>
  </property>

</configuration><commit_msg>Add static group mapping for Hadoop groups<commit_after><?xml version="1.0"?>
<!--
################################################
#
#              Generated by Chef
#
################################################
-->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value><%= node['bcpc']['hadoop']['hdfs_url']%></value>
  </property>

  <property>
    <name>hadoop.proxyuser.httpfs.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.httpfs.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hue.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.hue.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.oozie.hosts</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.proxyuser.oozie.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>io.compression.codecs</name>
    <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
  </property>

  <property>
    <name>mapreduce.cluster.local.dir</name>
    <value><%=@mounts.map{ |d| "file:///disk/#{d}/yarn/mapred-local" }.join(",")%></value>
  </property>

  <property>
     <name>yarn.nodemanager.log-dirs</name>
    <value><%=@mounts.map{ |d| "file:///disk/#{d}/yarn/logs" }.join(",")%></value>
  </property>

  <property>
    <name>yarn.nodemanager.linux-container-executor.group</name>
    <value>yarn</value>
  </property>

  <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>

  <property>
    <name>net.topology.script.file.name</name>
    <value>/etc/hadoop/conf/topology</value>
  </property>

  <property>
    <name>io.compression.codec.lzo.class</name>
    <value>com.hadoop.compression.lzo.LzoCodec</value>
  </property>

  <% if node[:bcpc][:hadoop][:kerberos][:enable] == true then %>
  <property>
    <name>hadoop.security.authentication</name>
    <value>kerberos</value>
    <description>Set theauthentication for the cluster. Valid values are: simple or kerberos.</description>
  </property>

  <property>
    <name>hadoop.security.authorization</name>
    <value>true</value>
    <description>Enable authorization for different protocols.</description>
  </property>

  <property>
    <name>hadoop.security.auth_to_local</name>
    <value>
      <% node[:bcpc][:hadoop][:kerberos][:data].each do|ke,va| %>
      RULE:[2:$1@$0](<%= va['principal']%>@.*<%= node[:bcpc][:hadoop][:kerberos][:realm] %>)s/.*/<%=va['owner'] %>/
      <% end %>
      DEFAULT
    </value>
    <description>
      The mapping from kerberos principal names to local OS user names.
    </description>
  </property>
  <% end %>

  <property>
    <name>hadoop.proxyuser.hive.hosts</name>
    <value>*</value>
  </property>
  
  <property>
    <name>hadoop.proxyuser.hive.groups</name>
    <value>*</value>
  </property>

  <property>
    <name>hadoop.user.group.static.mapping.overrides</name>
    <value>hdfs=hadoop,hdfs;yarn=mapred,hadoop;</value>
  </property>
</configuration>
{'fraction_non_alphanumeric': 0.22503082614056721, 'fraction_numerical': 0.0018495684340320592, 'mean_word_length': 3.8317200297840657, 'pattern_counts': {'":': 0, '<': 279, '<?xml version=': 2, '>': 279, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2274559', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2293, 'n_words': 281}