<commit_before>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.util.xml;

import dk.statsbiblioteket.util.Files;
import dk.statsbiblioteket.util.Profiler;
import dk.statsbiblioteket.util.Strings;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import javax.xml.stream.*;
import java.io.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

public class XMLStepperTest extends TestCase {
    private static Log log = LogFactory.getLog(XMLStepperTest.class);

    private static final String SAMPLE =
            "<foo><bar xmlns=\"http://www.example.com/bar_ns/\">"
            + "<nam:subsub xmlns:nam=\"http://example.com/subsub_ns\">content1<!-- Sub comment --></nam:subsub>"
            + "<!-- Comment --></bar>\n"
            + "<bar><subsub>content2</subsub></bar></foo>";

    private static final String DERIVED_NAMESPACE =
            "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></foo>";

    private static final String OUTER_SNIPPET = "<bar>bar1</bar><bar>bar2</bar>";
    private static final String OUTER_FULL = "<major><foo>" + OUTER_SNIPPET + "</foo>\n<foo>next</foo></major>";

    private XMLInputFactory xmlFactory = XMLInputFactory.newInstance();
    {
        xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, true);
        // No resolving of external DTDs
        xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
    }
    private XMLOutputFactory xmlOutFactory = XMLOutputFactory.newInstance();


    public void testIsWellformed() {
        final String[] FINE = new String[]{
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></foo>",
                "<foo/>"
        };
        final String[] FAULTY = new String[]{
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar<bar></bar></foo>",
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar>",
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></doo>",
                "<foo xmlns=\"http://www.example.com/foo_ns/><bar>simple bar</bar></foo>",
        };
        for (String fine: FINE) {
            assertTrue("The XML should be well-formed: " + fine, XMLStepper.isWellformed(fine));
        }
        for (String faulty: FAULTY) {
            assertFalse("The XML should not be well-formed: " + faulty, XMLStepper.isWellformed(faulty));
        }

    }

    private final static String LIMIT_BARS =
            "<foo><bar zoo=\"true\"></bar><bar zoo=\"true\"></bar><bar zoo=\"false\"></bar><baz></baz></foo>";

    public void testLimitXMLSimple() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><baz /></foo>", false, true, false,
                    "/foo/bar", 0);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><baz /></foo>", false, true, false,
                    "/foo/bar", 1);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><bar zoo=\"true\" /><baz /></foo>", false, true, false,
                    "/foo/bar", 2);
    }

    public void testLimitPositiveList() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /></foo>", false, true, true,
                    "/foo$", -1, "/foo/bar", 1);
        assertLimit(LIMIT_BARS, "<foo><baz /></foo>", false, true, true,
                    "/foo$", -1, "/foo/baz", 1);
    }

    public void testLimitXMLAttribute() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"false\" /><baz /></foo>", false, false, false,
                    "/foo/bar#zoo=true", 0);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><bar zoo=\"false\" /><baz /></foo>", false, false, false,
                    "/foo/bar#zoo=true", 1);
    }

    public void testLimitCountPatterns() throws XMLStreamException {
        assertLimit(LIMIT_BARS,
                    "<foo><bar zoo=\"true\"></bar><bar zoo=\"true\"></bar><baz></baz></foo>", true, true, false,
                    ".*", 2);
    }

    // Limits on specific field with specific tag
    public void testLimitPerformance() throws IOException, XMLStreamException {
        final String SAMPLE = getSample(9423);
        final int RUNS = 10;
        final Map<Pattern, Integer> limits = new HashMap<Pattern, Integer>();
        limits.put(Pattern.compile("/record/datafield#tag=Z30"), 10);

        Profiler profiler = new Profiler(RUNS);

        String reduced = "";
        for (int run = 0 ; run < RUNS ; run++) {
            reduced = XMLStepper.limitXML(SAMPLE, limits, false, false, false);
            profiler.beat();
        }
        log.info(String.format(
                "Reduced %d blocks @ %dKB to %dKB at %.1f reductions/sec",
                RUNS, SAMPLE.length() / 1024, reduced.length() / 1024, profiler.getBps(false)));
        assertTrue("The reduced XML should contain datafields after the skipped ones",
                   reduced.contains("<datafield tag=\"LOC\""));
    }

    // Limits in all datafields, counting on unique datafield#tag=value
    public void testLimitPerformanceCountPatterns() throws IOException, XMLStreamException {
        final String SAMPLE = getSample(9423);
        final int RUNS = 10;
        final Map<Pattern, Integer> limits = new HashMap<Pattern, Integer>();
        limits.put(Pattern.compile("/record/datafield#tag=.*"), 10);

        Profiler profiler = new Profiler(RUNS);

        XMLStepper.Limiter limiter = XMLStepper.createLimiter(limits, true, false, false);

        String reduced = "";
        for (int run = 0 ; run < RUNS ; run++) {
            reduced = limiter.limit(SAMPLE);
            profiler.beat();
        }
        log.info(String.format(<commit_msg>Testing: Added test for exception throwing on problems limiting XML<commit_after>/*
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */
package dk.statsbiblioteket.util.xml;

import dk.statsbiblioteket.util.Files;
import dk.statsbiblioteket.util.Profiler;
import dk.statsbiblioteket.util.Strings;
import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import javax.xml.stream.*;
import java.io.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

public class XMLStepperTest extends TestCase {
    private static Log log = LogFactory.getLog(XMLStepperTest.class);

    private static final String SAMPLE =
            "<foo><bar xmlns=\"http://www.example.com/bar_ns/\">"
            + "<nam:subsub xmlns:nam=\"http://example.com/subsub_ns\">content1<!-- Sub comment --></nam:subsub>"
            + "<!-- Comment --></bar>\n"
            + "<bar><subsub>content2</subsub></bar></foo>";

    private static final String DERIVED_NAMESPACE =
            "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></foo>";

    private static final String OUTER_SNIPPET = "<bar>bar1</bar><bar>bar2</bar>";
    private static final String OUTER_FULL = "<major><foo>" + OUTER_SNIPPET + "</foo>\n<foo>next</foo></major>";

    private XMLInputFactory xmlFactory = XMLInputFactory.newInstance();
    {
        xmlFactory.setProperty(XMLInputFactory.IS_COALESCING, true);
        // No resolving of external DTDs
        xmlFactory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
    }
    private XMLOutputFactory xmlOutFactory = XMLOutputFactory.newInstance();


    public void testIsWellformed() {
        final String[] FINE = new String[]{
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></foo>",
                "<foo/>"
        };
        final String[] FAULTY = new String[]{
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar<bar></bar></foo>",
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar>",
                "<foo xmlns=\"http://www.example.com/foo_ns/\"><bar>simple bar</bar></doo>",
                "<foo xmlns=\"http://www.example.com/foo_ns/><bar>simple bar</bar></foo>",
        };
        for (String fine: FINE) {
            assertTrue("The XML should be well-formed: " + fine, XMLStepper.isWellformed(fine));
        }
        for (String faulty: FAULTY) {
            assertFalse("The XML should not be well-formed: " + faulty, XMLStepper.isWellformed(faulty));
        }

    }

    private final static String LIMIT_BARS =
            "<foo><bar zoo=\"true\"></bar><bar zoo=\"true\"></bar><bar zoo=\"false\"></bar><baz></baz></foo>";

    public void testLimitXMLSimple() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><baz /></foo>", false, true, false,
                    "/foo/bar", 0);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><baz /></foo>", false, true, false,
                    "/foo/bar", 1);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><bar zoo=\"true\" /><baz /></foo>", false, true, false,
                    "/foo/bar", 2);
    }

    public void testLimitPositiveList() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /></foo>", false, true, true,
                    "/foo$", -1, "/foo/bar", 1);
        assertLimit(LIMIT_BARS, "<foo><baz /></foo>", false, true, true,
                    "/foo$", -1, "/foo/baz", 1);
    }

    public void testLimitXMLAttribute() throws XMLStreamException {
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"false\" /><baz /></foo>", false, false, false,
                    "/foo/bar#zoo=true", 0);
        assertLimit(LIMIT_BARS, "<foo><bar zoo=\"true\" /><bar zoo=\"false\" /><baz /></foo>", false, false, false,
                    "/foo/bar#zoo=true", 1);
    }

    public void testLimitCountPatterns() throws XMLStreamException {
        assertLimit(LIMIT_BARS,
                    "<foo><bar zoo=\"true\"></bar><bar zoo=\"true\"></bar><baz></baz></foo>", true, true, false,
                    ".*", 2);
    }

    // Limits on specific field with specific tag
    public void testLimitPerformance() throws IOException, XMLStreamException {
        final String SAMPLE = getSample(9423);
        final int RUNS = 10;
        final Map<Pattern, Integer> limits = new HashMap<Pattern, Integer>();
        limits.put(Pattern.compile("/record/datafield#tag=Z30"), 10);

        Profiler profiler = new Profiler(RUNS);

        String reduced = "";
        for (int run = 0 ; run < RUNS ; run++) {
            reduced = XMLStepper.limitXML(SAMPLE, limits, false, false, false);
            profiler.beat();
        }
        log.info(String.format(
                "Reduced %d blocks @ %dKB to %dKB at %.1f reductions/sec",
                RUNS, SAMPLE.length() / 1024, reduced.length() / 1024, profiler.getBps(false)));
        assertTrue("The reduced XML should contain datafields after the skipped ones",
                   reduced.contains("<datafield tag=\"LOC\""));
    }

    public void testLimitException() throws XMLStreamException {
        Map<Pattern, Integer> lims = new HashMap<Pattern, Integer>();
        lims.put(Pattern.compile("/foo/bar"), 1);
        XMLStreamReader in = xmlFactory.createXMLStreamReader(new StringReader("<foo><bar s=\"t\" /><<</foo>"));
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        XMLStreamWriter out = xmlOutFactory.createXMLStreamWriter(os);
        try {
            XMLStepper.limitXML(in, out, lims, true, true, false);
            fail("An XMLStreamException was expected here due to invalid input XML");
        } catch (XMLStreamException e) {
            // Intended
        }
    }

    // Limits in all datafields, counting on unique datafield#tag=value
    public void testLimitPerformanceCountPatterns() throws IOException, XMLStreamException {
        final String SAMPLE = getSample(9423);
        final int RUNS = 10;
        final Map<Pattern, Integer> limits = new HashMap<Pattern, Integer>();
        limits.put(Pattern.compile("/record/datafield#tag=.*"), 10);

        Profiler profiler = new Profiler(RUNS);

        XMLStepper.Limiter limiter = XMLStepper.createLimiter(limits, true, false, false);

        String reduced = "";
        for (int run = 0 ; run < RUNS ; run++) {
            reduced = limiter.limit(SAMPLE);
            profiler.beat();
        }
        log.info(String.format(
{'fraction_non_alphanumeric': 0.15208732851453016, 'fraction_numerical': 0.007006933175984658, 'mean_word_length': 2.6725352112676055, 'pattern_counts': {'":': 0, '<': 208, '<?xml version=': 0, '>': 200, 'https://': 0, 'lorem ipsum': 0, 'www.': 16, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4942739', 'n_tokens_mistral': 4120, 'n_tokens_neox': 3874, 'n_words': 985}