<commit_before>package dk.statsbiblioteket.medieplatform.newspaper.manualQA;

import dk.statsbiblioteket.medieplatform.autonomous.ResultCollector;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.AttributeParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.eventhandlers.DefaultTreeEventHandler;
import dk.statsbiblioteket.medieplatform.newspaper.manualQA.flagging.FlaggingCollector;
import dk.statsbiblioteket.util.Pair;

/**
 * At the time of implementing this, all histograms we have seen had a very big spike for a color number 0, 1, or 2. It seems
 * this cannot be avoided. This is not a spike in the normal curve-wise sense. It is a single color that juts up as a pin in the
 * graph. The same could be the case at the other end. If this spike is too high, say 2% of the total number of pixels, the image
 * should be flagged.
 */
public class EndSpikeHistogramChecker extends DefaultTreeEventHandler {
    private dk.statsbiblioteket.medieplatform.newspaper.manualQA.flagging.FlaggingCollector flaggingCollector;
    private ResultCollector resultCollector;
    private double threshold;

    private int minColorConsideredBlack;
    private int maxColorConsideredBlack;
    private int minColorConsideredWhite;
    private int maxColorConsideredWhite;
    private double maxPercentAllowedNearBlack;
    private double maxPercentAllowedNearWhite;

    public EndSpikeHistogramChecker(ResultCollector resultCollector, FlaggingCollector flaggingCollector, double threshold,
                                    int minColorConsideredBlack, int maxColorConsideredBlack, int minColorConsideredWhite,
                                    int maxColorConsideredWhite, double maxPercentAllowedNearBlack,
                                    double maxPercentAllowedNearWhite) {
        this.flaggingCollector = flaggingCollector;
        this.resultCollector = resultCollector;
        this.threshold = threshold;
        this.minColorConsideredBlack = minColorConsideredBlack;
        this.maxColorConsideredBlack = maxColorConsideredBlack;
        this.minColorConsideredWhite = minColorConsideredWhite;
        this.maxColorConsideredWhite = maxColorConsideredWhite;
        this.maxPercentAllowedNearBlack = maxPercentAllowedNearBlack;
        this.maxPercentAllowedNearWhite = maxPercentAllowedNearWhite;
    }

    @Override
    public void handleAttribute(AttributeParsingEvent event) {
        try {
            if (event.getName().endsWith(".film.histogram.xml")) {
                Histogram histogram = new Histogram(event.getData());
                Pair<Spike, Long> spikeAndTotal = findHighestSpike(histogram);
                Spike spike = spikeAndTotal.getLeft();
                Long total = spikeAndTotal.getRight();
                double amount = (spike.getValue() + 0.0) / total;
                if (amount > threshold) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found suspicious color spike for color '" + spike.getColor() + "'. " + amount * 100
                                    + "% of the graph is contained in this spike.");
                }

                double lowLightPercent = percentColorsInRange(minColorConsideredBlack, maxColorConsideredBlack, histogram);
                double highLightPercent = percentColorsInRange(minColorConsideredWhite, maxColorConsideredWhite, histogram);
                String lowColorForPrint = minColorConsideredBlack
                        + (minColorConsideredBlack == maxColorConsideredBlack ? "" : "-" + maxColorConsideredBlack);
                String highColorForPrint = minColorConsideredWhite
                        + (minColorConsideredWhite == maxColorConsideredWhite ? "" : "-" + maxColorConsideredWhite);

                if (lowLightPercent > maxPercentAllowedNearBlack) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found possible low-light blowout: more than " + maxPercentAllowedNearBlack
                                    + "% pixels of color " + lowColorForPrint + ", found " + lowLightPercent + "%");
                }

                if (highLightPercent > maxPercentAllowedNearWhite) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found possible high-light blowout: more than " + maxPercentAllowedNearWhite
                                    + "% pixels of color " + highColorForPrint + ", found " + highLightPercent + "%");
                }
            }
        } catch (Exception e) {
            resultCollector.addFailure(event.getName(), "exception", getComponent(), e.getMessage());
        }

    }

    private double percentColorsInRange(int minColor, int maxColor, Histogram histogram) {
        double totalColorCount = 0;
        double intervalColorCount = 0;

        long[] values = histogram.values();
        for (int i = 0; i < values.length; i++) {
            totalColorCount += values[i];
            if (minColor <= i && i <= maxColor) {
                intervalColorCount += values[i];
            }
        }
<commit_msg>Documentation<commit_after>package dk.statsbiblioteket.medieplatform.newspaper.manualQA;

import dk.statsbiblioteket.medieplatform.autonomous.ResultCollector;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.AttributeParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.eventhandlers.DefaultTreeEventHandler;
import dk.statsbiblioteket.medieplatform.newspaper.manualQA.flagging.FlaggingCollector;
import dk.statsbiblioteket.util.Pair;

/**
 * At the time of implementing this, all histograms we have seen had a very big spike for a color number 0, 1, or 2. It seems
 * this cannot be avoided. This is not a spike in the normal curve-wise sense. It is a single color that juts up as a pin in the
 * graph. The same could be the case at the other end. If this spike is too high, say 2% of the total number of pixels, the image
 * should be flagged.
 */
public class EndSpikeHistogramChecker extends DefaultTreeEventHandler {
    private dk.statsbiblioteket.medieplatform.newspaper.manualQA.flagging.FlaggingCollector flaggingCollector;
    private ResultCollector resultCollector;
    private double threshold;

    private int minColorConsideredBlack;
    private int maxColorConsideredBlack;
    private int minColorConsideredWhite;
    private int maxColorConsideredWhite;
    private double maxPercentAllowedNearBlack;
    private double maxPercentAllowedNearWhite;

    public EndSpikeHistogramChecker(ResultCollector resultCollector, FlaggingCollector flaggingCollector, double threshold,
                                    int minColorConsideredBlack, int maxColorConsideredBlack, int minColorConsideredWhite,
                                    int maxColorConsideredWhite, double maxPercentAllowedNearBlack,
                                    double maxPercentAllowedNearWhite) {
        this.flaggingCollector = flaggingCollector;
        this.resultCollector = resultCollector;
        this.threshold = threshold;
        this.minColorConsideredBlack = minColorConsideredBlack;
        this.maxColorConsideredBlack = maxColorConsideredBlack;
        this.minColorConsideredWhite = minColorConsideredWhite;
        this.maxColorConsideredWhite = maxColorConsideredWhite;
        this.maxPercentAllowedNearBlack = maxPercentAllowedNearBlack;
        this.maxPercentAllowedNearWhite = maxPercentAllowedNearWhite;
    }

    @Override
    public void handleAttribute(AttributeParsingEvent event) {
        try {
            if (event.getName().endsWith(".film.histogram.xml")) {
                Histogram histogram = new Histogram(event.getData());
                Pair<Spike, Long> spikeAndTotal = findHighestSpike(histogram);
                Spike spike = spikeAndTotal.getLeft();
                Long total = spikeAndTotal.getRight();
                double amount = (spike.getValue() + 0.0) / total;
                if (amount > threshold) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found suspicious color spike for color '" + spike.getColor() + "'. " + amount * 100
                                    + "% of the graph is contained in this spike.");
                }

                double lowLightPercent = percentColorsInRange(minColorConsideredBlack, maxColorConsideredBlack, histogram);
                double highLightPercent = percentColorsInRange(minColorConsideredWhite, maxColorConsideredWhite, histogram);
                String lowColorForPrint = minColorConsideredBlack
                        + (minColorConsideredBlack == maxColorConsideredBlack ? "" : "-" + maxColorConsideredBlack);
                String highColorForPrint = minColorConsideredWhite
                        + (minColorConsideredWhite == maxColorConsideredWhite ? "" : "-" + maxColorConsideredWhite);

                if (lowLightPercent > maxPercentAllowedNearBlack) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found possible low-light blowout: more than " + maxPercentAllowedNearBlack
                                    + "% pixels of color " + lowColorForPrint + ", found " + lowLightPercent + "%");
                }

                if (highLightPercent > maxPercentAllowedNearWhite) {
                    flaggingCollector.addFlag(
                            event,
                            "jp2file",
                            getComponent(),
                            "Found possible high-light blowout: more than " + maxPercentAllowedNearWhite
                                    + "% pixels of color " + highColorForPrint + ", found " + highLightPercent + "%");
                }
            }
        } catch (Exception e) {
            resultCollector.addFailure(event.getName(), "exception", getComponent(), e.getMessage());
        }

    }

    /**
     * Calculate the percentage of the total amount of pixels in the given histogram that lie in the given range.
     * @param minColor The lowest (leftmost) position in the wanted range of the histogram.
     * @param maxColor The highest (rightmost) position in the wanted range of the histogram.
     * @param histogram The histogram wherein the given range lies
     * @return The percentage of the total amount of pixels in the given histogram that lie in the given range.
     */
    private double percentColorsInRange(int minColor, int maxColor, Histogram histogram) {
        double totalColorCount = 0;
        double intervalColorCount = 0;

        long[] values = histogram.values();
        for (int i = 0; i < values.length; i++) {
            totalColorCount += values[i];
            if (minColor <= i && i <= maxColor) {
                intervalColorCount += values[i];
            }
        }

{'fraction_non_alphanumeric': 0.06987591305113086, 'fraction_numerical': 0.0026401478482795037, 'mean_word_length': 2.2201756871635023, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '988300', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2713, 'n_words': 709}