<commit_before><?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

<!-- Auto detect the brisk job tracker -->
<property>
  <name>mapred.job.tracker</name>
  <value>${brisk.job.tracker}</value>
  <description>
    The address of the job tracker
  </description>                                                                                                      
</property>

<property>
  <name>mapred.child.java.opts</name>
  <value>-Xmx256m</value>
  <description>
    Tune your mapred jvm arguments for best performance. 
    Also see documentation from jvm vendor.
  </description>
</property>

<!-- Enable Hue plugins -->
<property>
  <name>mapred.jobtracker.plugins</name>
  <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
  <description>Comma-separated list of jobtracker plug-ins to be activated.
  </description>
</property>
<property>
  <name>jobtracker.thrift.address</name>
  <value>0.0.0.0:9290</value>
</property>
<!-- FairScheduler is included. Uncomment to enable. -->
<!-- 
<property> 
  <name>mapred.jobtracker.taskScheduler</name> 
  <value>org.apache.hadoop.mapred.FairScheduler</value> 
</property>
<property> 
  <name>mapred.fairscheduler.poolnameproperty</name> 
  <value>mapred.job.queue.name</value> 
</property>
<property> 
  <name>mapred.fairscheduler.allocation.file</name> 
  <value>resources/hadoop/conf/fairscheduler-allocations.xml</value> 
</property>
 -->
</configuration>
<commit_msg>improved defaults for brisk mapred-site.xml<commit_after><?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>

<!-- Auto detect the brisk job tracker -->
<property>
  <name>mapred.job.tracker</name>
  <value>${brisk.job.tracker}</value>
  <description>
    The address of the job tracker
  </description>                                                                                                      
</property>


<!-- performance defaults (not required) -->
<property>
    <name>mapred.job.reuse.jvm.num.tasks</name>
    <value>24</value>
</property>

<!-- Compress intermediate files for better performance -->
<property>
    <name>mapred.compress.map.output</name>
    <value>true</value>
</property>

<property>
  <name>mapred.map.output.compression.codec</name>
  <value>com.hadoop.compression.snappy.SnappyCodec</value>
</property>

<property>
  <name>mapred.child.java.opts</name>
  <value>-Xmx256m</value>
  <description>
    Tune your mapred jvm arguments for best performance. 
    Also see documentation from jvm vendor.
  </description>
</property>


<!-- Enable Hue plugins -->
<property>
  <name>mapred.jobtracker.plugins</name>
  <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
  <description>Comma-separated list of jobtracker plug-ins to be activated.
  </description>
</property>
<property>
  <name>jobtracker.thrift.address</name>
  <value>0.0.0.0:9290</value>
</property>


<!-- FairScheduler is included. Uncomment to enable. -->
<!-- 
<property> 
  <name>mapred.jobtracker.taskScheduler</name> 
  <value>org.apache.hadoop.mapred.FairScheduler</value> 
</property>
<property> 
  <name>mapred.fairscheduler.poolnameproperty</name> 
  <value>mapred.job.queue.name</value> 
</property>
<property> 
  <name>mapred.fairscheduler.allocation.file</name> 
  <value>resources/hadoop/conf/fairscheduler-allocations.xml</value> 
</property>
 -->
</configuration>

{'fraction_non_alphanumeric': 0.15469146238377007, 'fraction_numerical': 0.007889546351084813, 'mean_word_length': 3.2926239419588876, 'pattern_counts': {'":': 0, '<': 137, '<?xml version=': 2, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1672563', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1112, 'n_words': 206}