<commit_before><configuration>

  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>
  
  <appender name="FILE" class="ch.qos.logback.core.FileAppender">
    <file>/data/latis/log/latis.log</file>
    <append>true</append>
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>
  
  <appender name="ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>/data/latis/log/mms.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>/data/latis/log/mms.%d{yyyy-MM}.log</fileNamePattern>
      <maxHistory>12</maxHistory>
    </rollingPolicy>
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>

  <root level="debug">
    <appender-ref ref="FILE" />
    <appender-ref ref="STDOUT" />
  </root>
</configuration><commit_msg>save the debug noise for when we really need it<commit_after><configuration>

  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>
  
  <appender name="FILE" class="ch.qos.logback.core.FileAppender">
    <file>/data/latis/log/latis.log</file>
    <append>true</append>
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>
  
  <appender name="ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>/data/latis/log/mms.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>/data/latis/log/mms.%d{yyyy-MM}.log</fileNamePattern>
      <maxHistory>12</maxHistory>
    </rollingPolicy>
    <encoder>
      <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS, GMT} %-5level %logger{36} (%thread\)] %msg%n</pattern>
    </encoder>
  </appender>

  <root level="info">
    <appender-ref ref="FILE" />
    <appender-ref ref="STDOUT" />
  </root>
</configuration>
{'fraction_non_alphanumeric': 0.20406398616515348, 'fraction_numerical': 0.009511456982274102, 'mean_word_length': 3.902542372881356, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4799098', 'n_tokens_mistral': 933, 'n_tokens_neox': 898, 'n_words': 114}