<commit_before>package dk.statsbiblioteket.newspaper.metadatachecker;

import dk.statsbiblioteket.medieplatform.autonomous.ResultCollector;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.AttributeParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.DataFileNodeBeginsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.DataFileNodeEndsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.NodeBeginsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.NodeEndParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.eventhandlers.DefaultTreeEventHandler;
import dk.statsbiblioteket.newspaper.metadatachecker.caches.DocumentCache;
import dk.statsbiblioteket.util.Strings;
import dk.statsbiblioteket.util.xml.DOM;
import dk.statsbiblioteket.util.xml.XPathSelector;
import org.w3c.dom.Document;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayDeque;

import static dk.statsbiblioteket.util.Strings.getStackTrace;

/** This class tests that the mix and alto files agree on the image size */
public class AltoMixCrossCheckEventHandler extends DefaultTreeEventHandler {

    private final ResultCollector resultCollector;
    private XPathSelector xpath;
    private ArrayDeque<SizeSet> sizesStack = new ArrayDeque<>();
    DocumentCache documentCache;

    public AltoMixCrossCheckEventHandler(ResultCollector resultCollector, DocumentCache documentCache) {
        this.resultCollector = resultCollector;
        this.documentCache = documentCache;

        xpath = DOM.createXPathSelector(
                "a", "http://www.loc.gov/standards/alto/ns-v2#", "mix", "http://www.loc.gov/mix/v20");

    }

    /**
     * Push a sizes object onto the stack
     *
     * @param event
     */
    @Override
    public void handleNodeBegin(NodeBeginsParsingEvent event) {
        if (!(event instanceof DataFileNodeBeginsParsingEvent)) {
            sizesStack.push(new SizeSet(event.getName()));
        }
    }

    /**
     * Pop the stack and verify that the sizes match for that folder
     *
     * @param event
     */
    @Override
    public void handleNodeEnd(NodeEndParsingEvent event) {
        if (!(event instanceof DataFileNodeEndsParsingEvent)) {
            SizeSet sizes = sizesStack.pop();
            verify(sizes, resultCollector);
        }
    }

    /**
     * Compare the alto sizes and the mix sizes, if both are > 0. If less than 0, we assume that they have not been
     * set (ie. there was no alto file or something)
     *
     * @param sizes           the store of sizes
     * @param resultCollector the result collector
     */
    private void verify(SizeSet sizes, ResultCollector resultCollector) {
        if (!sizes.getMixSize()
                  .equalWidth(sizes.getAltoSize())) {
            resultCollector.addFailure(
                    sizes.getFolder(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7: The file '" + sizes.getFolder() + ".mix.xml' and file '" + sizes.getFolder() + ".alto.xml' should agree on height");

        }
        if (!sizes.getMixSize()
                  .equalHeight(sizes.getAltoSize())) {
            resultCollector.addFailure(
                    sizes.getFolder(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7: The file '" + sizes.getFolder() + ".mix.xml' and file '" + sizes.getFolder() + ".alto.xml' should agree on width");
        }
    }

    @Override
    public void handleAttribute(AttributeParsingEvent event) {

        SizeSet sizes = sizesStack.peek();
        try {
            if (event.getName()
                     .endsWith(".alto.xml")) {

                extractAltoSizes(documentCache.getDocument(event, true), sizes.getAltoSize(), event.getName());

            } else if (event.getName()
                            .endsWith(".mix.xml")) {
                extractMixSizes(documentCache.getDocument(event, true), sizes.getMixSize(), event.getName());
            }


        } catch (IOException e) {
            resultCollector.addFailure(
                    event.getName(), "exception", getClass().getSimpleName(), "Error processing metadata: " + e.toString(),
                    getStackTrace(e));

        } catch (NumberFormatException e) {
            resultCollector.addFailure(
                    event.getName(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7 / 2K-9 - 2K-10: Failed to parse integer: " + e.toString(),
                    Strings.getStackTrace(e));
        }

    }

    private Document asDom(InputStream data) {
        return DOM.streamToDOM(data, true);
    }
<commit_msg>Removed unused code<commit_after>package dk.statsbiblioteket.newspaper.metadatachecker;

import dk.statsbiblioteket.medieplatform.autonomous.ResultCollector;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.AttributeParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.DataFileNodeBeginsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.DataFileNodeEndsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.NodeBeginsParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.common.NodeEndParsingEvent;
import dk.statsbiblioteket.medieplatform.autonomous.iterator.eventhandlers.DefaultTreeEventHandler;
import dk.statsbiblioteket.newspaper.metadatachecker.caches.DocumentCache;
import dk.statsbiblioteket.util.Strings;
import dk.statsbiblioteket.util.xml.DOM;
import dk.statsbiblioteket.util.xml.XPathSelector;
import org.w3c.dom.Document;

import java.io.IOException;
import java.util.ArrayDeque;

import static dk.statsbiblioteket.util.Strings.getStackTrace;

/** This class tests that the mix and alto files agree on the image size */
public class AltoMixCrossCheckEventHandler extends DefaultTreeEventHandler {

    private final ResultCollector resultCollector;
    private XPathSelector xpath;
    private ArrayDeque<SizeSet> sizesStack = new ArrayDeque<>();
    DocumentCache documentCache;

    public AltoMixCrossCheckEventHandler(ResultCollector resultCollector, DocumentCache documentCache) {
        this.resultCollector = resultCollector;
        this.documentCache = documentCache;

        xpath = DOM.createXPathSelector(
                "a", "http://www.loc.gov/standards/alto/ns-v2#", "mix", "http://www.loc.gov/mix/v20");

    }

    /**
     * Push a sizes object onto the stack
     *
     * @param event
     */
    @Override
    public void handleNodeBegin(NodeBeginsParsingEvent event) {
        if (!(event instanceof DataFileNodeBeginsParsingEvent)) {
            sizesStack.push(new SizeSet(event.getName()));
        }
    }

    /**
     * Pop the stack and verify that the sizes match for that folder
     *
     * @param event
     */
    @Override
    public void handleNodeEnd(NodeEndParsingEvent event) {
        if (!(event instanceof DataFileNodeEndsParsingEvent)) {
            SizeSet sizes = sizesStack.pop();
            verify(sizes, resultCollector);
        }
    }

    /**
     * Compare the alto sizes and the mix sizes, if both are > 0. If less than 0, we assume that they have not been
     * set (ie. there was no alto file or something)
     *
     * @param sizes           the store of sizes
     * @param resultCollector the result collector
     */
    private void verify(SizeSet sizes, ResultCollector resultCollector) {
        if (!sizes.getMixSize()
                  .equalWidth(sizes.getAltoSize())) {
            resultCollector.addFailure(
                    sizes.getFolder(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7: The file '" + sizes.getFolder() + ".mix.xml' and file '" + sizes.getFolder() + ".alto.xml' should agree on height");

        }
        if (!sizes.getMixSize()
                  .equalHeight(sizes.getAltoSize())) {
            resultCollector.addFailure(
                    sizes.getFolder(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7: The file '" + sizes.getFolder() + ".mix.xml' and file '" + sizes.getFolder() + ".alto.xml' should agree on width");
        }
    }

    @Override
    public void handleAttribute(AttributeParsingEvent event) {

        SizeSet sizes = sizesStack.peek();
        try {
            if (event.getName()
                     .endsWith(".alto.xml")) {

                extractAltoSizes(documentCache.getDocument(event, true), sizes.getAltoSize(), event.getName());

            } else if (event.getName()
                            .endsWith(".mix.xml")) {
                extractMixSizes(documentCache.getDocument(event, true), sizes.getMixSize(), event.getName());
            }


        } catch (IOException e) {
            resultCollector.addFailure(
                    event.getName(), "exception", getClass().getSimpleName(), "Error processing metadata: " + e.toString(),
                    getStackTrace(e));

        } catch (NumberFormatException e) {
            resultCollector.addFailure(
                    event.getName(),
                    "metadata",
                    getClass().getSimpleName(),
                    "2J-7 / 2K-9 - 2K-10: Failed to parse integer: " + e.toString(),
                    Strings.getStackTrace(e));
        }

    }

{'fraction_non_alphanumeric': 0.10268694022078734, 'fraction_numerical': 0.003540928973130598, 'mean_word_length': 2.4894622093023258, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7064509', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2660, 'n_words': 551}