<commit_before>package com.linkedin.thirdeye.bootstrap;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.linkedin.thirdeye.api.StarTreeConfig;
import com.linkedin.thirdeye.api.StarTreeConstants;
import com.linkedin.thirdeye.api.StarTreeRecord;
import com.linkedin.thirdeye.impl.StarTreeRecordStreamAvroFileImpl;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class StarTreeRecordAnalysisTool implements Runnable
{
  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
  private static final TypeReference TYPE_REFERENCE = new TypeReference<Map<String, String>>(){};

  private final PrintWriter printWriter;
  private final StarTreeConfig config;
  private final Map<String, String> filter;
  private final Long time;
  private final List<Iterable<StarTreeRecord>> streams;
  private final boolean debug;

  public StarTreeRecordAnalysisTool(PrintWriter printWriter,
                                    StarTreeConfig config,
                                    Map<String, String> filter,
                                    Long time,
                                    List<Iterable<StarTreeRecord>> streams,
                                    boolean debug)
  {
    this.printWriter = printWriter;
    this.config = config;
    this.filter = filter;
    this.time = time;
    this.streams = streams;
    this.debug = debug;
  }

  @Override
  public void run()
  {
    Map<String, Integer> aggregates = new HashMap<String, Integer>();
    for (String metricName : config.getMetricNames())
    {
      aggregates.put(metricName, 0);
    }

    for (Iterable<StarTreeRecord> stream : streams)
    {
      for (StarTreeRecord record : stream)
      {
        if (matches(record))
        {
          updateAggregates(record, aggregates);

          if (debug)
          {
            printWriter.println(record);
          }
        }
      }
    }

    printWriter.println("Aggregates:");
    for (String metricName : config.getMetricNames())
    {
      printWriter.print("\t");
      printWriter.print(metricName);
      printWriter.print("=");
      printWriter.print(aggregates.get(metricName));<commit_msg>better output for record analysis tool<commit_after>package com.linkedin.thirdeye.bootstrap;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.linkedin.thirdeye.api.StarTreeConfig;
import com.linkedin.thirdeye.api.StarTreeConstants;
import com.linkedin.thirdeye.api.StarTreeRecord;
import com.linkedin.thirdeye.impl.StarTreeRecordStreamAvroFileImpl;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class StarTreeRecordAnalysisTool implements Runnable
{
  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
  private static final TypeReference TYPE_REFERENCE = new TypeReference<Map<String, String>>(){};

  private final PrintWriter printWriter;
  private final StarTreeConfig config;
  private final Map<String, String> filter;
  private final Long time;
  private final List<Iterable<StarTreeRecord>> streams;
  private final boolean debug;

  public StarTreeRecordAnalysisTool(PrintWriter printWriter,
                                    StarTreeConfig config,
                                    Map<String, String> filter,
                                    Long time,
                                    List<Iterable<StarTreeRecord>> streams,
                                    boolean debug)
  {
    this.printWriter = printWriter;
    this.config = config;
    this.filter = filter;
    this.time = time;
    this.streams = streams;
    this.debug = debug;
  }

  @Override
  public void run()
  {
    Map<String, Integer> aggregates = new HashMap<String, Integer>();
    for (String metricName : config.getMetricNames())
    {
      aggregates.put(metricName, 0);
    }

    for (Iterable<StarTreeRecord> stream : streams)
    {
      for (StarTreeRecord record : stream)
      {
        if (matches(record))
        {
          updateAggregates(record, aggregates);

          if (debug)
          {
            printWriter.println(record);
          }
        }
      }
    }

    for (String metricName : config.getMetricNames())
    {
      printWriter.print(metricName);
      printWriter.print("=");
      printWriter.print(aggregates.get(metricName));
{'fraction_non_alphanumeric': 0.09308719560094265, 'fraction_numerical': 0.001178318931657502, 'mean_word_length': 2.8730038022813686, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6445276', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1406, 'n_words': 304}