<commit_before>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.xwiki.rendering.test.cts;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;

import org.apache.commons.configuration2.CompositeConfiguration;
import org.apache.commons.configuration2.builder.fluent.Configurations;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.reflections.Reflections;
import org.reflections.scanners.Scanners;
import org.reflections.util.ClasspathHelper;
import org.reflections.util.ConfigurationBuilder;
import org.reflections.util.FilterBuilder;

/**
 * Finds all test files in the current classloader, read them and return test data to represent them. See
 * {@link CompatibilityTestSuite} for a description of the algorithm.
 *
 * @version $Id$
 * @since 4.1M1
 * @see CompatibilityTestSuite
 */
public class TestDataParser
{
    /**
     * Represents the Java resource separator.
     */
    private static final String SLASH = "/";

    /**
     * Represents the dot package separator.
     */
    private static final String DOT = ".";

    /**
     * Read all test data. See {@link CompatibilityTestSuite} for a detailed explanation of the algorithm.
     *
     * @param syntaxId the id of the syntax for which to parse data for
     * @param ctsRootPackageName the root of the CTS resources
     * @param packageFilter the regex to filter packages
     * @param pattern a regex to decide which {@code *.xml} resources should be found. The default should be to find
     *            them all
     * @return the list of test data
     * @throws Exception in case of error while reading test data
     */
    public List<TestData> parseTestData(String syntaxId, String ctsRootPackageName, String packageFilter,
        String pattern) throws Exception
    {
        ClassLoader classLoader = getClass().getClassLoader();

        List<TestData> data = new ArrayList<>();
        String syntaxDirectory = computeSyntaxDirectory(syntaxId);

        // Read the suite-level Configuration data.
        TestDataConfiguration configuration = parseTestConfiguration(syntaxDirectory, ctsRootPackageName, classLoader);

        Set<String> relativeDirectoryNames = findRelativeTestDirectoryNames(ctsRootPackageName, packageFilter, pattern);
        for (String relativeDirectoryName : relativeDirectoryNames) {
            List<TestData> testDatas = parseSingleTestData(syntaxDirectory, ctsRootPackageName, relativeDirectoryName,
                configuration, classLoader);
            for (TestData testData : testDatas) {
                testData.syntaxId = syntaxId;
                testData.prefix = relativeDirectoryName;
                testData.configuration = configuration;
                data.add(testData);
            }
        }
        return data;
    }

    /**
     * Parse data for single test.
     *
     * @param syntaxDirectory the syntax directory from where to read syntax test data (eg "xwiki20" for "xwiki/2.0"
     *            syntax)
     * @param ctsRootPackageName the root of the CTS resources
     * @param relativeDirectoryName the name of the relative directory for a CTS test (eg "/simple/bold/bold1")
     * @param configuration the test configuration
     * @param classLoader the class loader from which the test data is read from
     * @return the TestData instances for both input and output tests, including possible input alias tests
     * @throws IOException in case of error while reading test data
     */
    public List<TestData> parseSingleTestData(String syntaxDirectory, String ctsRootPackageName,
        String relativeDirectoryName, TestDataConfiguration configuration, ClassLoader classLoader) throws IOException
    {
        // Look for syntax-specific input/output file and read their content
        TestData testDataIN = new TestData();
        testDataIN.isSyntaxInputTest = true;
        TestData testDataOUT = new TestData();

        // Look for CTS input/output file and read their contents
        Pair<Pair<String, String>, Pair<String, String>> ctsData =
            readDataForPrefix(ctsRootPackageName + SLASH + relativeDirectoryName, "xml", classLoader);

        Pair<Pair<String, String>, Pair<String, String>> syntaxData = readDataForPrefix(
            syntaxDirectory + SLASH + relativeDirectoryName, configuration.fileExtension, classLoader);

        testDataIN.syntaxData = syntaxData.getLeft().getLeft();
        testDataIN.syntaxExtension = syntaxData.getLeft().getRight();
        testDataIN.ctsData = ctsData.getRight().getLeft();
        testDataIN.ctsExtension = ctsData.getRight().getRight();
        testDataOUT.syntaxData = syntaxData.getRight().getLeft();
        testDataOUT.syntaxExtension = syntaxData.getRight().getRight();
        testDataOUT.ctsData = ctsData.getLeft().getLeft();
        testDataOUT.ctsExtension = ctsData.getLeft().getRight();

        // Read possible "in" aliases
        List<TestData> testDataINAliases =
            parseAliasesTestData(syntaxDirectory, relativeDirectoryName, ctsData, configuration, classLoader);

        // If the inherit configuration property is set and if the returned syntax is empty load from the inherit
        // syntax.
        if (configuration.inheritSyntax != null) {
            Pair<Pair<String, String>, Pair<String, String>> inheritedSyntaxData =
                readDataForPrefix(computeSyntaxDirectory(configuration.inheritSyntax) + SLASH + relativeDirectoryName,
                    configuration.fileExtension, classLoader);
            if (testDataIN.syntaxData == null) {
                testDataIN.syntaxData = inheritedSyntaxData.getLeft().getLeft();
                testDataIN.syntaxExtension = inheritedSyntaxData.getLeft().getRight();
            }
            if (testDataOUT.syntaxData == null) {
                testDataOUT.syntaxData = inheritedSyntaxData.getRight().getLeft();
                testDataOUT.syntaxExtension = inheritedSyntaxData.getRight().getRight();
            }
        }

        List<TestData> result = new ArrayList<>();
        result.add(testDataIN);
        result.addAll(testDataINAliases);
        result.add(testDataOUT);
        return result;
    }

    /**
     * Parse Alias test data for inputs.
     *
     * @param syntaxDirectory the syntax directory from where to read syntax test data (eg "xwiki20" for "xwiki/2.0"
     *            syntax)
     * @param relativeDirectoryName the name of the relative directory for a CTS test (eg "/simple/bold/bold1")
     * @param ctsData the CTS data to use to construct the alias test data
     * @param configuration the test configuration
     * @param classLoader the class loader from which the test data is read from
     * @return the TestData instances for both input and output tests, including possible input alias tests
     * @throws IOException in case of error while reading test data
     */
    private List<TestData> parseAliasesTestData(String syntaxDirectory, String relativeDirectoryName,<commit_msg>XRENDERING-639: Allow recursive inheritance for the CTS<commit_after>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.xwiki.rendering.test.cts;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;

import org.apache.commons.configuration2.CompositeConfiguration;
import org.apache.commons.configuration2.builder.fluent.Configurations;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.reflections.Reflections;
import org.reflections.scanners.Scanners;
import org.reflections.util.ClasspathHelper;
import org.reflections.util.ConfigurationBuilder;
import org.reflections.util.FilterBuilder;

/**
 * Finds all test files in the current classloader, read them and return test data to represent them. See
 * {@link CompatibilityTestSuite} for a description of the algorithm.
 *
 * @version $Id$
 * @since 4.1M1
 * @see CompatibilityTestSuite
 */
public class TestDataParser
{
    /**
     * Represents the Java resource separator.
     */
    private static final String SLASH = "/";

    /**
     * Represents the dot package separator.
     */
    private static final String DOT = ".";

    /**
     * Read all test data. See {@link CompatibilityTestSuite} for a detailed explanation of the algorithm.
     *
     * @param syntaxId the id of the syntax for which to parse data for
     * @param ctsRootPackageName the root of the CTS resources
     * @param packageFilter the regex to filter packages
     * @param pattern a regex to decide which {@code *.xml} resources should be found. The default should be to find
     *            them all
     * @return the list of test data
     * @throws Exception in case of error while reading test data
     */
    public List<TestData> parseTestData(String syntaxId, String ctsRootPackageName, String packageFilter,
        String pattern) throws Exception
    {
        ClassLoader classLoader = getClass().getClassLoader();

        List<TestData> data = new ArrayList<>();
        String syntaxDirectory = computeSyntaxDirectory(syntaxId);

        // Read the suite-level Configuration data.
        TestDataConfiguration configuration = parseTestConfiguration(syntaxDirectory, ctsRootPackageName, classLoader);

        Set<String> relativeDirectoryNames = findRelativeTestDirectoryNames(ctsRootPackageName, packageFilter, pattern);
        for (String relativeDirectoryName : relativeDirectoryNames) {
            List<TestData> testDatas = parseSingleTestData(syntaxDirectory, ctsRootPackageName, relativeDirectoryName,
                configuration, classLoader);
            for (TestData testData : testDatas) {
                testData.syntaxId = syntaxId;
                testData.prefix = relativeDirectoryName;
                testData.configuration = configuration;
                data.add(testData);
            }
        }
        return data;
    }

    /**
     * Parse data for single test.
     *
     * @param syntaxDirectory the syntax directory from where to read syntax test data (eg "xwiki20" for "xwiki/2.0"
     *            syntax)
     * @param ctsRootPackageName the root of the CTS resources
     * @param relativeDirectoryName the name of the relative directory for a CTS test (eg "/simple/bold/bold1")
     * @param configuration the test configuration
     * @param classLoader the class loader from which the test data is read from
     * @return the TestData instances for both input and output tests, including possible input alias tests
     * @throws IOException in case of error while reading test data
     */
    public List<TestData> parseSingleTestData(String syntaxDirectory, String ctsRootPackageName,
        String relativeDirectoryName, TestDataConfiguration configuration, ClassLoader classLoader) throws IOException
    {
        // Look for syntax-specific input/output file and read their content
        TestData testDataIN = new TestData();
        testDataIN.isSyntaxInputTest = true;
        TestData testDataOUT = new TestData();

        // Look for CTS input/output file and read their contents
        Pair<Pair<String, String>, Pair<String, String>> ctsData =
            readDataForPrefix(ctsRootPackageName + SLASH + relativeDirectoryName, "xml", classLoader);

        Pair<Pair<String, String>, Pair<String, String>> syntaxData = readDataForPrefix(
            syntaxDirectory + SLASH + relativeDirectoryName, configuration.fileExtension, classLoader);

        testDataIN.syntaxData = syntaxData.getLeft().getLeft();
        testDataIN.syntaxExtension = syntaxData.getLeft().getRight();
        testDataIN.ctsData = ctsData.getRight().getLeft();
        testDataIN.ctsExtension = ctsData.getRight().getRight();
        testDataOUT.syntaxData = syntaxData.getRight().getLeft();
        testDataOUT.syntaxExtension = syntaxData.getRight().getRight();
        testDataOUT.ctsData = ctsData.getLeft().getLeft();
        testDataOUT.ctsExtension = ctsData.getLeft().getRight();

        // Read possible "in" aliases
        List<TestData> testDataINAliases =
            parseAliasesTestData(syntaxDirectory, relativeDirectoryName, ctsData, configuration, classLoader);

        // If the inherit configuration property is set and if the returned syntax is empty load from the inherit
        // syntax.
        TestDataConfiguration currentConfiguration = configuration;
        while ((testDataIN.syntaxData == null || testDataOUT.syntaxData == null)
            && currentConfiguration.inheritSyntax != null) {
            final String inheritedSyntaxDirectory = computeSyntaxDirectory(currentConfiguration.inheritSyntax);

            Pair<Pair<String, String>, Pair<String, String>> inheritedSyntaxData =
                readDataForPrefix(inheritedSyntaxDirectory + SLASH + relativeDirectoryName,
                    configuration.fileExtension, classLoader);
            if (testDataIN.syntaxData == null) {
                testDataIN.syntaxData = inheritedSyntaxData.getLeft().getLeft();
                testDataIN.syntaxExtension = inheritedSyntaxData.getLeft().getRight();
            }
            if (testDataOUT.syntaxData == null) {
                testDataOUT.syntaxData = inheritedSyntaxData.getRight().getLeft();
                testDataOUT.syntaxExtension = inheritedSyntaxData.getRight().getRight();
            }

            try {
                currentConfiguration = parseTestConfiguration(inheritedSyntaxDirectory, ctsRootPackageName,
                    classLoader);
            } catch (Exception ignored) {
                break;
            }
        }

        List<TestData> result = new ArrayList<>();
        result.add(testDataIN);
        result.addAll(testDataINAliases);
        result.add(testDataOUT);
        return result;
    }

    /**
     * Parse Alias test data for inputs.
     *
     * @param syntaxDirectory the syntax directory from where to read syntax test data (eg "xwiki20" for "xwiki/2.0"
     *            syntax)
     * @param relativeDirectoryName the name of the relative directory for a CTS test (eg "/simple/bold/bold1")
     * @param ctsData the CTS data to use to construct the alias test data
     * @param configuration the test configuration
     * @param classLoader the class loader from which the test data is read from
     * @return the TestData instances for both input and output tests, including possible input alias tests
     * @throws IOException in case of error while reading test data
     */
    private List<TestData> parseAliasesTestData(String syntaxDirectory, String relativeDirectoryName,
{'fraction_non_alphanumeric': 0.07762774373921837, 'fraction_numerical': 0.0038665159716852063, 'mean_word_length': 3.1644785731979193, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3637453', 'n_tokens_mistral': 4252, 'n_tokens_neox': 4121, 'n_words': 1433}