<commit_before><?xml version="1.0" encoding="utf-8"?>
<configuration>
  <!--
    Compromise on data locality, but gain on reducing the number of map slots occupied by
    using 2x block size split sizes.
  -->
  <property>
    <name>mapred.max.split.size</name>
    <value>256000000</value>
  </property>

  <!--
    Use the Hive thrift metastore.
    TODO: this will change when we move to the Hive2 server and CDH5
  -->
  <property>
    <name>hive.metastore.local</name>
    <value>false</value>
  </property>
  <property>
    <name>hive.metastore.uris</name>
    <value>${hive.metastore.uris}</value>
  </property>

  <!-- Explicit, to allow us to debug query plans etc -->
  <property>
    <name>hive.exec.scratchdir</name>
    <value>/tmp/hive-occurrence.download</value>
  </property>

  <!--
    The following are to declare the hive output directory and set permissions explicitly, to enable it to be
    used with confidence in other parts of the workflow outside of Hive.
  -->
  <property>
    <name>hive.metastore.warehouse.dir</name>
    <value>${occurrence.download.hive.hdfs.out}</value>
  </property>
  <property>
    <name>hive.warehouse.subdir.inherit.perms</name>
    <value>true</value>
  </property>
  <property>
    <name>hive.metastore.execute.setugi</name>
    <value>true</value><commit_msg>Remove obsolete metastore configuration.<commit_after><?xml version="1.0" encoding="utf-8"?>
<configuration>
  <!--
    Compromise on data locality, but gain on reducing the number of map slots occupied by
    using 2x block size split sizes.
  -->
  <property>
    <name>mapred.max.split.size</name>
    <value>256000000</value>
  </property>

  <!--
    Use the Hive thrift metastore.
  -->
  <property>
    <name>hive.metastore.uris</name>
    <value>${hive.metastore.uris}</value>
  </property>

  <!-- Explicit, to allow us to debug query plans etc -->
  <property>
    <name>hive.exec.scratchdir</name>
    <value>/tmp/hive-occurrence.download</value>
  </property>

  <!--
    The following are to declare the hive output directory and set permissions explicitly, to enable it to be
    used with confidence in other parts of the workflow outside of Hive.
  -->
  <property>
    <name>hive.metastore.warehouse.dir</name>
    <value>${occurrence.download.hive.hdfs.out}</value>
  </property>
  <property>
    <name>hive.warehouse.subdir.inherit.perms</name>
    <value>true</value>
  </property>
  <property>
    <name>hive.metastore.execute.setugi</name>
    <value>true</value>
{'fraction_non_alphanumeric': 0.1442845723421263, 'fraction_numerical': 0.011191047162270184, 'mean_word_length': 3.2786324786324785, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 2, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '826467', 'n_tokens_mistral': 863, 'n_tokens_neox': 831, 'n_words': 203}