<commit_before><configuration scan="true">
  <property name="SERVICE" value="shard-v2" />
  <property name="LOG_DIR" value="/var/log/precog" />

  <appender name="SERVERLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>${LOG_DIR}/${SERVICE}.server.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <!-- daily rollover -->
      <fileNamePattern>${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log</fileNamePattern>

      <!-- keep 30 days' worth of history -->
      <maxHistory>30</maxHistory>
    </rollingPolicy>
    <encoder>
      <pattern>%d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n</pattern>
    </encoder>
  </appender>

  <logger name="blueeyes.core.http" level="trace" />
  <!--
  <logger name="com.precog.yggdrasil.actor.KafkaShardIngestActor" level="debug" />
  <logger name="com.precog.yggdrasil.actor.ProjectionsActor" level="debug" />
  <logger name="com.precog.yggdrasil.jdbm3.JDBMProjection" level="debug" />
  -->

  <root level="trace">
    <appender-ref ref="SERVERLOG" />
  </root>
</configuration>
<commit_msg>Suppres BlueEyes logging noise for dev<commit_after><configuration scan="true">
  <property name="SERVICE" value="shard-v2" />
  <property name="LOG_DIR" value="/var/log/precog" />

  <appender name="SERVERLOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>${LOG_DIR}/${SERVICE}.server.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <!-- daily rollover -->
      <fileNamePattern>${LOG_DIR}/${SERVICE}.server-%d{yyyyMMdd}.log</fileNamePattern>

      <!-- keep 30 days' worth of history -->
      <maxHistory>30</maxHistory>
    </rollingPolicy>
    <encoder>
      <pattern>%d{ISO8601} [%.10thread] %.-1level %logger{20} {%X} - %msg%n</pattern>
    </encoder>
  </appender>

  <logger name="blueeyes.core" level="info" />
  <!--
  <logger name="com.precog.yggdrasil.actor.KafkaShardIngestActor" level="debug" />
  <logger name="com.precog.yggdrasil.actor.ProjectionsActor" level="debug" />
  <logger name="com.precog.yggdrasil.jdbm3.JDBMProjection" level="debug" />
  -->

  <root level="trace">
    <appender-ref ref="SERVERLOG" />
  </root>
</configuration>

{'fraction_non_alphanumeric': 0.1912617030762372, 'fraction_numerical': 0.013374944271065538, 'mean_word_length': 4.135011441647597, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7543075', 'n_tokens_mistral': 807, 'n_tokens_neox': 790, 'n_words': 109}