<commit_before>package com.bbn.bue.common.evaluation;

import com.bbn.bue.common.Inspector;
import com.bbn.bue.common.symbols.Symbol;

import com.google.common.annotations.Beta;
import com.google.common.io.CharSink;

import java.io.IOException;

import static com.google.common.base.Preconditions.checkNotNull;

/**
 * Observes an {@link Alignment} between system items on the left and gold standard items on the
 * right. A system/gold item is assumed to be found if aligned.  Writes summary F-measure scores
 * based on its observations.
 *
 * At the moment this only writes confusion matrices but it will be extended to write more soon.
 *
 * Although this only implements {@code Inspector<Alignment<Object,Object>>}, it is safe to cast it
 * to inspect any alignment.
 */
@Beta
public final class AggregateBinaryFScoresInspector<KeyT, TestT>
    implements Inspector<Alignment<? extends KeyT, ? extends TestT>> {

  private final SummaryConfusionMatrices.Builder summaryConfusionMatrixB =
      SummaryConfusionMatrices.builder();
  private final CharSink outSink;
  private static final Symbol PRESENT = Symbol.from("Present");
  private static final Symbol ABSENT = Symbol.from("Absent");

  private AggregateBinaryFScoresInspector(final CharSink outSink) {
    this.outSink = checkNotNull(outSink);
  }

  public static <KeyT, TestT> AggregateBinaryFScoresInspector<KeyT, TestT> createOutputtingTo(
      CharSink outputSink) {
    return new AggregateBinaryFScoresInspector<KeyT, TestT>(outputSink);
  }

  @Override
  public void finish() throws IOException {
    final SummaryConfusionMatrix summaryConfusionMatrix = summaryConfusionMatrixB.build();
    outSink.write(SummaryConfusionMatrices.prettyPrint(summaryConfusionMatrix));
  }

  @Override
  public void inspect(final Alignment<? extends KeyT, ? extends TestT> alignment) {
    summaryConfusionMatrixB
        .accumulatePredictedGold(PRESENT, PRESENT, alignment.rightAligned().size());
    summaryConfusionMatrixB
        .accumulatePredictedGold(PRESENT, ABSENT, alignment.leftUnaligned().size());
    summaryConfusionMatrixB
        .accumulatePredictedGold(ABSENT, PRESENT, alignment.rightUnaligned().size());
  }
}
<commit_msg>Make AggregateF actually produce an F-score.<commit_after>package com.bbn.bue.common.evaluation;

import com.bbn.bue.common.Inspector;
import com.bbn.bue.common.symbols.Symbol;

import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharSink;

import java.io.IOException;

import static com.google.common.base.Preconditions.checkNotNull;

/**
 * Observes an {@link Alignment} between system items on the left and gold standard items on the
 * right. A system/gold item is assumed to be found if aligned.  Writes summary F-measure scores
 * based on its observations.
 *
 * At the moment this only writes confusion matrices but it will be extended to write more soon.
 *
 * Although this only implements {@code Inspector<Alignment<Object,Object>>}, it is safe to cast it
 * to inspect any alignment.
 */
@Beta
public final class AggregateBinaryFScoresInspector<KeyT, TestT>
    implements Inspector<Alignment<? extends KeyT, ? extends TestT>> {

  private final SummaryConfusionMatrices.Builder summaryConfusionMatrixB =
      SummaryConfusionMatrices.builder();
  private final ImmutableList.Builder<FMeasureCounts> countsBuilder;
  private final CharSink outSink;
  private static final Symbol PRESENT = Symbol.from("Present");
  private static final Symbol ABSENT = Symbol.from("Absent");

  private AggregateBinaryFScoresInspector(final CharSink outSink) {
    this.outSink = checkNotNull(outSink);
    countsBuilder = ImmutableList.builder();
  }

  public static <KeyT, TestT> AggregateBinaryFScoresInspector<KeyT, TestT> createOutputtingTo(
      CharSink outputSink) {
    return new AggregateBinaryFScoresInspector<KeyT, TestT>(outputSink);
  }

  @Override
  public void finish() throws IOException {
    final SummaryConfusionMatrix summaryConfusionMatrix = summaryConfusionMatrixB.build();
    final FMeasureCounts summaryCounts = FMeasureCounts.combineToMicroFMeasure(countsBuilder.build());
    outSink.write(SummaryConfusionMatrices.prettyPrint(summaryConfusionMatrix)
        + summaryCounts.compactPrettyString());
  }

  @Override
  public void inspect(final Alignment<? extends KeyT, ? extends TestT> alignment) {
    final int truePositives = alignment.rightAligned().size();
    final int falseNegatives = alignment.leftUnaligned().size();
    final int falsePositives = alignment.rightUnaligned().size();
    summaryConfusionMatrixB
        .accumulatePredictedGold(PRESENT, PRESENT, truePositives);
    summaryConfusionMatrixB
        .accumulatePredictedGold(PRESENT, ABSENT, falseNegatives);
    summaryConfusionMatrixB
        .accumulatePredictedGold(ABSENT, PRESENT, falsePositives);
    countsBuilder.add(FMeasureCounts.from(truePositives, falsePositives, falseNegatives));
  }
}

{'fraction_non_alphanumeric': 0.08588093322606596, 'fraction_numerical': 0.0, 'mean_word_length': 5.0869033047735615, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2898338', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1401, 'n_words': 369}