<commit_before>package com.bbn.kbp.events2014.scorer.observers;

import com.bbn.bue.common.annotations.MoveToBUECommon;
import com.bbn.bue.common.collections.MapUtils;
import com.bbn.bue.common.diff.FMeasureTableRenderer;
import com.bbn.bue.common.diff.ProvenancedConfusionMatrix;
import com.bbn.bue.common.diff.SummaryConfusionMatrix;
import com.bbn.bue.common.evaluation.FMeasureCounts;
import com.bbn.bue.common.io.GZIPByteSink;
import com.bbn.bue.common.scoring.Scored;
import com.bbn.bue.common.scoring.Scoreds;
import com.bbn.bue.common.serialization.jackson.JacksonSerializer;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.symbols.SymbolUtils;
import com.bbn.kbp.events2014.AssessedResponse;
import com.bbn.kbp.events2014.EventArgScoringAlignment;
import com.bbn.kbp.events2014.TypeRoleFillerRealis;
import com.bbn.kbp.events2014.scorer.SystemOutputEquivalenceClasses;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BreakdownComputer;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BreakdownFunctions;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BrokenDownSummaryConfusionMatrix;
import com.bbn.kbp.events2014.scorer.observers.errorloggers.HTMLErrorRecorder;

import com.carrotsearch.hppc.DoubleArrayList;
import com.carrotsearch.hppc.cursors.DoubleCursor;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Ordering;
import com.google.common.io.ByteSink;
import com.google.common.io.Files;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;

public final class EAScoringObserver extends KBPScoringObserver<TypeRoleFillerRealis> {

  public static final Symbol PRESENT = Symbol.from("PRESENT");
  public static final Symbol ABSENT = Symbol.from("ABSENT");

  private static final ImmutableSetMultimap<String, Symbol> F_MEASURES_TO_PRINT =
      ImmutableSetMultimap.of("Present", PRESENT);

  private final Predicate<AssessedResponse> ResponseCorrect;
  private final Logger log;
  // DocumentResult is defined within this class
  private final List<DocumentResult> documentResults = Lists.newArrayList();
  private final BreakdownComputer<TypeRoleFillerRealis> breakdownComputer;

  // I/O variables
  private final HTMLErrorRecorder renderer;
  private final ImmutableList<Outputter> outputters;

  public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(
      final HTMLErrorRecorder renderer,
      Iterable<? extends Outputter> outputters) {
    return new EAScoringObserver("Standard",
        AssessedResponse.IsCorrectUpToInexactJustifications,
        renderer, BreakdownFunctions.StandardBreakdowns, outputters);
  }

  public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(
      final HTMLErrorRecorder renderer,
      final Map<String, Function<TypeRoleFillerRealis, Symbol>> additionalBreakdowns,
      final Iterable<? extends Outputter> outputters) {
    return new EAScoringObserver("Standard",
        AssessedResponse.IsCorrectUpToInexactJustifications,
        renderer, plusStandardBreakdowns(additionalBreakdowns), outputters);
  }

  public static Outputter createStandardOutputter() {
    return new StandardOutputter();
  }

  public static Outputter createBootstrapOutputter(int seed, int numSamples) {
    return new BootstrapOutputter(seed, numSamples);
  }

  public static Outputter createEquivalenceClassIDOutputter() {
    return new EquivalenceClassIDOutputter();
  }

  private static ImmutableMap<String, Function<TypeRoleFillerRealis, Symbol>>
  plusStandardBreakdowns(Map<String, Function<TypeRoleFillerRealis, Symbol>> additionalBreakdowns) {
    final ImmutableMap.Builder<String, Function<TypeRoleFillerRealis, Symbol>> breakdowns =
        ImmutableMap.builder();

    breakdowns.putAll(BreakdownFunctions.StandardBreakdowns);
    breakdowns.putAll(additionalBreakdowns);

    return breakdowns.build();
  }


  /**
   * Write the final scoring output to a directory.
   *
   * @param directory Directory to write output to.
   */
  @Override<commit_msg>Bugfix: score statistics were not reset between systems<commit_after>package com.bbn.kbp.events2014.scorer.observers;

import com.bbn.bue.common.annotations.MoveToBUECommon;
import com.bbn.bue.common.collections.MapUtils;
import com.bbn.bue.common.diff.FMeasureTableRenderer;
import com.bbn.bue.common.diff.ProvenancedConfusionMatrix;
import com.bbn.bue.common.diff.SummaryConfusionMatrix;
import com.bbn.bue.common.evaluation.FMeasureCounts;
import com.bbn.bue.common.io.GZIPByteSink;
import com.bbn.bue.common.scoring.Scored;
import com.bbn.bue.common.scoring.Scoreds;
import com.bbn.bue.common.serialization.jackson.JacksonSerializer;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.symbols.SymbolUtils;
import com.bbn.kbp.events2014.AssessedResponse;
import com.bbn.kbp.events2014.EventArgScoringAlignment;
import com.bbn.kbp.events2014.TypeRoleFillerRealis;
import com.bbn.kbp.events2014.scorer.SystemOutputEquivalenceClasses;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BreakdownComputer;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BreakdownFunctions;
import com.bbn.kbp.events2014.scorer.observers.breakdowns.BrokenDownSummaryConfusionMatrix;
import com.bbn.kbp.events2014.scorer.observers.errorloggers.HTMLErrorRecorder;

import com.carrotsearch.hppc.DoubleArrayList;
import com.carrotsearch.hppc.cursors.DoubleCursor;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Ordering;
import com.google.common.io.ByteSink;
import com.google.common.io.Files;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Iterables.transform;

public final class EAScoringObserver extends KBPScoringObserver<TypeRoleFillerRealis> {

  public static final Symbol PRESENT = Symbol.from("PRESENT");
  public static final Symbol ABSENT = Symbol.from("ABSENT");

  private static final ImmutableSetMultimap<String, Symbol> F_MEASURES_TO_PRINT =
      ImmutableSetMultimap.of("Present", PRESENT);

  private final Predicate<AssessedResponse> ResponseCorrect;
  private final Logger log;
  // DocumentResult is defined within this class
  private final List<DocumentResult> documentResults = Lists.newArrayList();
  private final BreakdownComputer<TypeRoleFillerRealis> breakdownComputer;

  // I/O variables
  private final HTMLErrorRecorder renderer;
  private final ImmutableList<Outputter> outputters;

  public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(
      final HTMLErrorRecorder renderer,
      Iterable<? extends Outputter> outputters) {
    return new EAScoringObserver("Standard",
        AssessedResponse.IsCorrectUpToInexactJustifications,
        renderer, BreakdownFunctions.StandardBreakdowns, outputters);
  }

  public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(
      final HTMLErrorRecorder renderer,
      final Map<String, Function<TypeRoleFillerRealis, Symbol>> additionalBreakdowns,
      final Iterable<? extends Outputter> outputters) {
    return new EAScoringObserver("Standard",
        AssessedResponse.IsCorrectUpToInexactJustifications,
        renderer, plusStandardBreakdowns(additionalBreakdowns), outputters);
  }

  public static Outputter createStandardOutputter() {
    return new StandardOutputter();
  }

  public static Outputter createBootstrapOutputter(int seed, int numSamples) {
    return new BootstrapOutputter(seed, numSamples);
  }

  public static Outputter createEquivalenceClassIDOutputter() {
    return new EquivalenceClassIDOutputter();
  }

  private static ImmutableMap<String, Function<TypeRoleFillerRealis, Symbol>>
  plusStandardBreakdowns(Map<String, Function<TypeRoleFillerRealis, Symbol>> additionalBreakdowns) {
    final ImmutableMap.Builder<String, Function<TypeRoleFillerRealis, Symbol>> breakdowns =
        ImmutableMap.builder();

    breakdowns.putAll(BreakdownFunctions.StandardBreakdowns);
    breakdowns.putAll(additionalBreakdowns);

    return breakdowns.build();
  }

  @Override
  public void startCorpus() {
    documentResults.clear();
    ;
  }

  /**
   * Write the final scoring output to a directory.
   *
   * @param directory Directory to write output to.
   */
  @Override
{'fraction_non_alphanumeric': 0.09135035433453063, 'fraction_numerical': 0.0073779244733521016, 'mean_word_length': 5.755409836065573, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2376242', 'n_tokens_mistral': 3249, 'n_tokens_neox': 3135, 'n_words': 563}