<commit_before>{%- set namenode_fqdn = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.namenode and not G@roles:cdh5.hadoop.standby', 'grains.items', 'compound').values()[0]['fqdn'] -%}
<?xml version="1.0"?>
<configuration>
    <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
    </property>
  <property>
     <name>mapreduce.jobhistory.webapp.address</name>
     <value>{{ namenode_fqdn }}:19888</value>
  </property>
  <property>
     <name>mapreduce.jobhistory.address</name>
     <value>{{ namenode_fqdn }}:10020</value>
  </property>
  <property>
     <name>yarn.app.mapreduce.am.staging-dir</name>
     <value>/user</value>
  </property>
  <property>
     <name>mapreduce.job.counters.max</name>
     <value>240</value>
  </property>
  <property>
    <name>mapreduce.tasktracker.map.tasks.maximum</name>
    <value>{{ pillar.cdh5.mapred.map_tasks }}</value>
  </property>
  <property>
    <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
    <value>{{ pillar.cdh5.mapred.reduce_tasks }}</value>
  </property>
  <property>
    <name>mapreduce.job.reduces</name>
    <value>{{ pillar.cdh5.mapred.reduces }}</value>
  </property>
  <property>
     <name>mapreduce.output.fileoutputformat.compress</name>
     <value>true</value>
  </property>
  <property>
     <name>mapreduce.output.fileoutputformat.compress.type</name>
     <value>BLOCK</value>
  </property>
  <property>
     <name>mapreduce.map.output.compress</name>
     <value>true</value>
  </property>
  <property>
     <name>mapreduce.reduce.speculative</name>
     <value>false</value>
   </property>
  <property>
    <name>mapreduce.output.fileoutputformat.compress.codec</name>
    <value>org.apache.hadoop.io.compress.SnappyCodec</value>
  </property>
  <property>
     <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
     <value>0.4</value>
  </property><commit_msg>Modified the mapred file output format to use default compression and map output to use snappy compression<commit_after>{%- set namenode_fqdn = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.namenode and not G@roles:cdh5.hadoop.standby', 'grains.items', 'compound').values()[0]['fqdn'] -%}
<?xml version="1.0"?>
<configuration>
    <property>
      <name>mapreduce.framework.name</name>
      <value>yarn</value>
    </property>
  <property>
     <name>mapreduce.jobhistory.webapp.address</name>
     <value>{{ namenode_fqdn }}:19888</value>
  </property>
  <property>
     <name>mapreduce.jobhistory.address</name>
     <value>{{ namenode_fqdn }}:10020</value>
  </property>
  <property>
     <name>yarn.app.mapreduce.am.staging-dir</name>
     <value>/user</value>
  </property>
  <property>
     <name>mapreduce.job.counters.max</name>
     <value>240</value>
  </property>
  <property>
    <name>mapreduce.tasktracker.map.tasks.maximum</name>
    <value>{{ pillar.cdh5.mapred.map_tasks }}</value>
  </property>
  <property>
    <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
    <value>{{ pillar.cdh5.mapred.reduce_tasks }}</value>
  </property>
  <property>
    <name>mapreduce.job.reduces</name>
    <value>{{ pillar.cdh5.mapred.reduces }}</value>
  </property>
  <property>
     <name>mapreduce.output.fileoutputformat.compress</name>
     <value>true</value>
  </property>
  <property>
     <name>mapreduce.output.fileoutputformat.compress.type</name>
     <value>BLOCK</value>
  </property>
  <property>
     <name>mapreduce.map.output.compress</name>
     <value>true</value>
  </property>
  <property>
     <name>mapreduce.reduce.speculative</name>
     <value>false</value>
   </property>
  <property>
    <name>mapreduce.map.output.compress.codec</name>
    <value>org.apache.hadoop.io.compress.SnappyCodec</value>
  </property>
  <property>
     <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
     <value>0.4</value>
  </property>
{'fraction_non_alphanumeric': 0.18107416879795396, 'fraction_numerical': 0.011764705882352941, 'mean_word_length': 3.84634448574969, 'pattern_counts': {'":': 0, '<': 175, '<?xml version=': 2, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6942707', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1409, 'n_words': 177}