<commit_before>package com.bbn.kbp.events2014.scorer.observers;

import com.bbn.bue.common.StringUtils;
import com.bbn.bue.common.collections.MapUtils;
import com.bbn.bue.common.diff.FMeasureTableRenderer;
import com.bbn.bue.common.diff.ProvenancedConfusionMatrix;
import com.bbn.bue.common.diff.SummaryConfusionMatrix;
import com.bbn.bue.common.evaluation.FMeasureCounts;
import com.bbn.bue.common.evaluation.FMeasureInfo;
import com.bbn.bue.common.scoring.Scored;
import com.bbn.bue.common.scoring.Scoreds;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.symbols.SymbolUtils;
import com.bbn.kbp.events2014.AsssessedResponse;
import com.bbn.kbp.events2014.Response;
import com.bbn.kbp.events2014.scorer.AnswerKeyAnswerSource;
import com.bbn.kbp.events2014.scorer.SystemOutputAnswerSource;
import com.bbn.kbp.events2014.TypeRoleFillerRealis;
import com.bbn.kbp.events2014.scorer.observers.errorloggers.HTMLErrorRecorder;
import com.google.common.base.*;
import com.google.common.collect.*;
import com.google.common.io.CharSink;
import com.google.common.io.Files;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Predicates.compose;
import static com.google.common.base.Predicates.equalTo;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.getFirst;

public final class StrictStandardScoringObserver extends KBPScoringObserver<TypeRoleFillerRealis> {
	public static final Symbol PRESENT = Symbol.from("PRESENT");
	public static final Symbol ABSENT = Symbol.from("ABSENT");

	private final Predicate<AsssessedResponse> ResponseCorrect;
	private final Logger log;
	private final SummaryConfusionMatrix.Builder corpusConfusionMatrixBuilder =
			SummaryConfusionMatrix.builder();
    private final Map<Symbol, SummaryConfusionMatrix.Builder> byTypeConfusionMatrices =
            Maps.newHashMap();
    private final Map<Symbol, SummaryConfusionMatrix.Builder> byRoleConfusionMatrices =
            Maps.newHashMap();
    private final HTMLErrorRecorder renderer;

	public static KBPScoringObserver<TypeRoleFillerRealis> strictScorer(final HTMLErrorRecorder renderer) {
		return new StrictStandardScoringObserver("Strict", AsssessedResponse.IsCompletelyCorrect,
			renderer);
	}

	public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(final HTMLErrorRecorder renderer) {
		return new StrictStandardScoringObserver("Standard", AsssessedResponse.IsCorrectUpToInexactJustifications,
			renderer);
	}

    @Override
    public void writeCorpusOutput(File directory) throws IOException {
        final FMeasureCounts corpusFMeasure = corpusConfusionMatrixBuilder.build().FMeasureVsAllOthers(PRESENT);
        Iterable<Map.Entry<String, Map<Symbol, SummaryConfusionMatrix.Builder>>> printModes = ImmutableMap.of(
                "Aggregate", ImmutableMap.of(Symbol.from("Aggregate"), corpusConfusionMatrixBuilder),
                "Type", byTypeConfusionMatrices, "Role", byRoleConfusionMatrices).entrySet();


        for (final Map.Entry<String, Map<Symbol, SummaryConfusionMatrix.Builder>> printMode : printModes) {
            final String modeName = printMode.getKey();
            final Map<String, SummaryConfusionMatrix> data =
                    MapUtils.copyWithTransformedEntries(printMode.getValue(),
                            Functions.toStringFunction(), SummaryConfusionMatrix.Builder.Build);

            final File scoringBreakdownFilename = new File(directory, modeName);

            writeScoringBreakdown(modeName, data, scoringBreakdownFilename);
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_PRECISION,
                    new File(directory, modeName + ".improvementsWithPerfectPrecision"));
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_RECALL,
                    new File(directory, modeName + ".improvementsWithPerfectRecall"));
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_EVERYTHING,
                    new File(directory, modeName + ".improvementsWithPerfectEverything"));
        }
    }

    private static final Function<SummaryConfusionMatrix, FMeasureCounts> FmeasureVsPresent = new Function<SummaryConfusionMatrix, FMeasureCounts>() {
        @Override
        public FMeasureCounts apply(SummaryConfusionMatrix input) {
            return input.FMeasureVsAllOthers(PRESENT);
        }
    };

    private void writeScoringBreakdown(String modeName, Map<String, SummaryConfusionMatrix> data, File scoringBreakdownFilename) throws IOException {

        final Map<String, FMeasureCounts> fMeasuresToPrint = Maps.transformValues(data,
                FmeasureVsPresent);

        final FMeasureTableRenderer tableRenderer = FMeasureTableRenderer.create()
                .setNameFieldLength(4 + MapUtils.longestKeyLength(fMeasuresToPrint));

        Files.asCharSink(scoringBreakdownFilename, Charsets.UTF_8)
            .write(modeName + "\n" + tableRenderer.render(fMeasuresToPrint));
    }

    private enum ImprovementMode  {
        PERFECT_PRECISION {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives(),
                        baseCounts.falsePositives()-improvingCounts.falsePositives(),
                        baseCounts.falseNegatives());
            }},
        PERFECT_RECALL {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives()+improvingCounts.truePositives(),
                        baseCounts.falsePositives(),
                        baseCounts.falseNegatives()-improvingCounts.falseNegatives());
            }},
        PERFECT_EVERYTHING {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives()+improvingCounts.falseNegatives(),
                        baseCounts.falsePositives()-improvingCounts.falsePositives(),
                        baseCounts.falseNegatives()-improvingCounts.falseNegatives());
        }};

        public abstract FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts);
    };

    private void writeImprovements(final String modeName, final FMeasureCounts corpusFMeasure,
                                   final Map<String, SummaryConfusionMatrix> data,
                                   final ImprovementMode improvementMode,
                                   final File outputFile) throws IOException
    {
        final List<Scored<String>> improvements = Lists.newArrayList();
        for (Map.Entry<String, SummaryConfusionMatrix> entry : data.entrySet()) {
            final FMeasureCounts improvedCounts = improvementMode.improve(corpusFMeasure,
                    entry.getValue().FMeasureVsAllOthers(PRESENT));
            final double F1Improvement = improvedCounts.F1() - corpusFMeasure.F1();
            improvements.add(Scored.from(entry.getKey(), F1Improvement));
        }

        Collections.sort(improvements, Scoreds.<String>ByScoreThenByItem().reverse());

        final StringBuilder sb = new StringBuilder();

        sb.append("Score changes for " + modeName + " when improvement " + improvementMode.toString() + " is applied\n");

        final int nameFieldSize = 4+ MapUtils.longestKeyLength(data);
        final String headerFormat = "%"+nameFieldSize + "s    Change in F1\n";
        final String entryFormat = "%"+nameFieldSize + "s    %6.3f\n";

        sb.append(String.format(headerFormat, modeName));
        sb.append(Strings.repeat("-", nameFieldSize+18)).append("\n");
        for (Scored<String> improvement : improvements) {
            sb.append(String.format(entryFormat, improvement.item(), improvement.score()));
        }

        Files.asCharSink(outputFile, Charsets.UTF_8).write(sb.toString());
    }


    public void appendSummaryHTML(final StringBuilder sb) {
		final SummaryConfusionMatrix corpusMatrix = corpusConfusionMatrixBuilder.build();

//		sb.append(KBPHTMLRenderer.confusionMatrix(name, corpusMatrix));
		sb.append(corpusMatrix.FMeasureVsAllOthers(Symbol.from("PRESENT")).compactPrettyString());
	}

	private StrictStandardScoringObserver(final String name,
		final Predicate<AsssessedResponse> ResponseCorrect, final HTMLErrorRecorder renderer)
	{
		super(name);
		this.ResponseCorrect = checkNotNull(ResponseCorrect);
		this.log = LoggerFactory.getLogger(name());
		this.renderer = checkNotNull(renderer);
	}

	private void observeDocumentConfusionMatrix(final ProvenancedConfusionMatrix<TypeRoleFillerRealis> matrix) {
		corpusConfusionMatrixBuilder.accumulate(matrix.buildSummaryMatrix());
        for (final Symbol eventType : FluentIterable.from(matrix.entries())
                .transform(TypeRoleFillerRealis.Type).toSet())
        {
            accumulateConfusionMatrix(byTypeConfusionMatrices, eventType,
                    matrix.filteredCopy(compose(equalTo(eventType), TypeRoleFillerRealis.Type)));<commit_msg>Bugfix: need to scale improvements in F-measure by 100.0 before outputting<commit_after>package com.bbn.kbp.events2014.scorer.observers;

import com.bbn.bue.common.StringUtils;
import com.bbn.bue.common.collections.MapUtils;
import com.bbn.bue.common.diff.FMeasureTableRenderer;
import com.bbn.bue.common.diff.ProvenancedConfusionMatrix;
import com.bbn.bue.common.diff.SummaryConfusionMatrix;
import com.bbn.bue.common.evaluation.FMeasureCounts;
import com.bbn.bue.common.evaluation.FMeasureInfo;
import com.bbn.bue.common.scoring.Scored;
import com.bbn.bue.common.scoring.Scoreds;
import com.bbn.bue.common.symbols.Symbol;
import com.bbn.bue.common.symbols.SymbolUtils;
import com.bbn.kbp.events2014.AsssessedResponse;
import com.bbn.kbp.events2014.Response;
import com.bbn.kbp.events2014.scorer.AnswerKeyAnswerSource;
import com.bbn.kbp.events2014.scorer.SystemOutputAnswerSource;
import com.bbn.kbp.events2014.TypeRoleFillerRealis;
import com.bbn.kbp.events2014.scorer.observers.errorloggers.HTMLErrorRecorder;
import com.google.common.base.*;
import com.google.common.collect.*;
import com.google.common.io.CharSink;
import com.google.common.io.Files;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Predicates.compose;
import static com.google.common.base.Predicates.equalTo;
import static com.google.common.collect.Iterables.any;
import static com.google.common.collect.Iterables.getFirst;

public final class StrictStandardScoringObserver extends KBPScoringObserver<TypeRoleFillerRealis> {
	public static final Symbol PRESENT = Symbol.from("PRESENT");
	public static final Symbol ABSENT = Symbol.from("ABSENT");

	private final Predicate<AsssessedResponse> ResponseCorrect;
	private final Logger log;
	private final SummaryConfusionMatrix.Builder corpusConfusionMatrixBuilder =
			SummaryConfusionMatrix.builder();
    private final Map<Symbol, SummaryConfusionMatrix.Builder> byTypeConfusionMatrices =
            Maps.newHashMap();
    private final Map<Symbol, SummaryConfusionMatrix.Builder> byRoleConfusionMatrices =
            Maps.newHashMap();
    private final HTMLErrorRecorder renderer;

	public static KBPScoringObserver<TypeRoleFillerRealis> strictScorer(final HTMLErrorRecorder renderer) {
		return new StrictStandardScoringObserver("Strict", AsssessedResponse.IsCompletelyCorrect,
			renderer);
	}

	public static KBPScoringObserver<TypeRoleFillerRealis> standardScorer(final HTMLErrorRecorder renderer) {
		return new StrictStandardScoringObserver("Standard", AsssessedResponse.IsCorrectUpToInexactJustifications,
			renderer);
	}

    @Override
    public void writeCorpusOutput(File directory) throws IOException {
        final FMeasureCounts corpusFMeasure = corpusConfusionMatrixBuilder.build().FMeasureVsAllOthers(PRESENT);
        Iterable<Map.Entry<String, Map<Symbol, SummaryConfusionMatrix.Builder>>> printModes = ImmutableMap.of(
                "Aggregate", ImmutableMap.of(Symbol.from("Aggregate"), corpusConfusionMatrixBuilder),
                "Type", byTypeConfusionMatrices, "Role", byRoleConfusionMatrices).entrySet();


        for (final Map.Entry<String, Map<Symbol, SummaryConfusionMatrix.Builder>> printMode : printModes) {
            final String modeName = printMode.getKey();
            final Map<String, SummaryConfusionMatrix> data =
                    MapUtils.copyWithTransformedEntries(printMode.getValue(),
                            Functions.toStringFunction(), SummaryConfusionMatrix.Builder.Build);

            final File scoringBreakdownFilename = new File(directory, modeName);

            writeScoringBreakdown(modeName, data, scoringBreakdownFilename);
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_PRECISION,
                    new File(directory, modeName + ".improvementsWithPerfectPrecision"));
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_RECALL,
                    new File(directory, modeName + ".improvementsWithPerfectRecall"));
            writeImprovements(modeName, corpusFMeasure, data, ImprovementMode.PERFECT_EVERYTHING,
                    new File(directory, modeName + ".improvementsWithPerfectEverything"));
        }
    }

    private static final Function<SummaryConfusionMatrix, FMeasureCounts> FmeasureVsPresent = new Function<SummaryConfusionMatrix, FMeasureCounts>() {
        @Override
        public FMeasureCounts apply(SummaryConfusionMatrix input) {
            return input.FMeasureVsAllOthers(PRESENT);
        }
    };

    private void writeScoringBreakdown(String modeName, Map<String, SummaryConfusionMatrix> data, File scoringBreakdownFilename) throws IOException {

        final Map<String, FMeasureCounts> fMeasuresToPrint = Maps.transformValues(data,
                FmeasureVsPresent);

        final FMeasureTableRenderer tableRenderer = FMeasureTableRenderer.create()
                .setNameFieldLength(4 + MapUtils.longestKeyLength(fMeasuresToPrint));

        Files.asCharSink(scoringBreakdownFilename, Charsets.UTF_8)
            .write(modeName + "\n" + tableRenderer.render(fMeasuresToPrint));
    }

    private enum ImprovementMode  {
        PERFECT_PRECISION {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives(),
                        baseCounts.falsePositives()-improvingCounts.falsePositives(),
                        baseCounts.falseNegatives());
            }},
        PERFECT_RECALL {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives()+improvingCounts.truePositives(),
                        baseCounts.falsePositives(),
                        baseCounts.falseNegatives()-improvingCounts.falseNegatives());
            }},
        PERFECT_EVERYTHING {
            public FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts) {
                return FMeasureCounts.from(baseCounts.truePositives()+improvingCounts.falseNegatives(),
                        baseCounts.falsePositives()-improvingCounts.falsePositives(),
                        baseCounts.falseNegatives()-improvingCounts.falseNegatives());
        }};

        public abstract FMeasureCounts improve(FMeasureCounts baseCounts, FMeasureCounts improvingCounts);
    };

    private void writeImprovements(final String modeName, final FMeasureCounts corpusFMeasure,
                                   final Map<String, SummaryConfusionMatrix> data,
                                   final ImprovementMode improvementMode,
                                   final File outputFile) throws IOException
    {
        final List<Scored<String>> improvements = Lists.newArrayList();
        for (Map.Entry<String, SummaryConfusionMatrix> entry : data.entrySet()) {
            final FMeasureCounts improvedCounts = improvementMode.improve(corpusFMeasure,
                    entry.getValue().FMeasureVsAllOthers(PRESENT));
            final double F1Improvement = improvedCounts.F1() - corpusFMeasure.F1();
            improvements.add(Scored.from(entry.getKey(), F1Improvement));
        }

        Collections.sort(improvements, Scoreds.<String>ByScoreThenByItem().reverse());

        final StringBuilder sb = new StringBuilder();

        sb.append("Score changes for " + modeName + " when improvement " + improvementMode.toString() + " is applied\n");

        final int nameFieldSize = 4+ MapUtils.longestKeyLength(data);
        final String headerFormat = "%"+nameFieldSize + "s    Change in F1\n";
        final String entryFormat = "%"+nameFieldSize + "s    %6.3f\n";

        sb.append(String.format(headerFormat, modeName));
        sb.append(Strings.repeat("-", nameFieldSize+18)).append("\n");
        for (Scored<String> improvement : improvements) {
            sb.append(String.format(entryFormat, improvement.item(), 100.0*improvement.score()));
        }

        Files.asCharSink(outputFile, Charsets.UTF_8).write(sb.toString());
    }


    public void appendSummaryHTML(final StringBuilder sb) {
		final SummaryConfusionMatrix corpusMatrix = corpusConfusionMatrixBuilder.build();

//		sb.append(KBPHTMLRenderer.confusionMatrix(name, corpusMatrix));
		sb.append(corpusMatrix.FMeasureVsAllOthers(Symbol.from("PRESENT")).compactPrettyString());
	}

	private StrictStandardScoringObserver(final String name,
		final Predicate<AsssessedResponse> ResponseCorrect, final HTMLErrorRecorder renderer)
	{
		super(name);
		this.ResponseCorrect = checkNotNull(ResponseCorrect);
		this.log = LoggerFactory.getLogger(name());
		this.renderer = checkNotNull(renderer);
	}

	private void observeDocumentConfusionMatrix(final ProvenancedConfusionMatrix<TypeRoleFillerRealis> matrix) {
		corpusConfusionMatrixBuilder.accumulate(matrix.buildSummaryMatrix());
        for (final Symbol eventType : FluentIterable.from(matrix.entries())
                .transform(TypeRoleFillerRealis.Type).toSet())
        {
            accumulateConfusionMatrix(byTypeConfusionMatrices, eventType,
                    matrix.filteredCopy(compose(equalTo(eventType), TypeRoleFillerRealis.Type)));
{'fraction_non_alphanumeric': 0.09143793960547887, 'fraction_numerical': 0.004971177746046856, 'mean_word_length': 3.804369918699187, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '382697', 'n_tokens_mistral': 5455, 'n_tokens_neox': 5132, 'n_words': 887}