<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.smoketest;

import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.http.HttpHost;
import org.apache.http.util.EntityUtils;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
import org.junit.After;
import org.junit.Before;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;

//The default 20 minutes timeout isn't always enough, please do not increase further than 30 before analyzing what makes this suite so slow
@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {

    public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
        super(testCandidate);
    }

    @ParametersFactory
    public static Iterable<Object[]> parameters() throws Exception {
        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS.size() + 1);
        entries.addAll(ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS);
        entries.add(new NamedXContentRegistry.Entry(ExecutableSection.class,
                new ParseField("compare_analyzers"), CompareAnalyzers::parse));
        NamedXContentRegistry executableSectionRegistry = new NamedXContentRegistry(entries);
        return ESClientYamlSuiteTestCase.createParameters(executableSectionRegistry);
    }

    @Override
    protected void afterIfFailed(List<Throwable> errors) {
        super.afterIfFailed(errors);
        String name = getTestName().split("=")[1];
        name = name.substring(0, name.length() - 1);
        name = name.replaceAll("/([^/]+)$", ".asciidoc:$1");
        logger.error("This failing test was generated by documentation starting at {}. It may include many snippets. "
                + "See docs/README.asciidoc for an explanation of test generation.", name);
    }

    @Override
    protected boolean randomizeContentType() {
        return false;
    }

    @Override
    protected ClientYamlTestClient initClientYamlTestClient(
            final ClientYamlSuiteRestSpec restSpec,
            final RestClient restClient,
            final List<HttpHost> hosts,
            final Version esVersion,
            final Version masterVersion) {
        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
    }

    @Before
    public void waitForRequirements() throws Exception {
        if (isCcrTest() || isGetLicenseTest()) {
            ESRestTestCase.waitForActiveLicense(adminClient());
        }
    }

    @After
    public void cleanup() throws Exception {
        if (isMachineLearningTest() || isTransformTest()) {
            ESRestTestCase.waitForPendingTasks(adminClient());
        }

        // check that there are no templates
        Request request = new Request("GET", "_cat/templates");
        request.addParameter("h", "name");
        String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity());
        if (false == "".equals(templates)) {
            for (String template : templates.split("\n")) {
                if (isXPackTemplate(template)) continue;
                if ("".equals(template)) {
                    throw new IllegalStateException("empty template in templates list:\n" + templates);
                }
                throw new RuntimeException("Template " + template + " not cleared after test");
            }
        }
    }

    @Override
    protected boolean preserveSLMPoliciesUponCompletion() {
        return isSLMTest() == false;
    }

    @Override
    protected boolean preserveILMPoliciesUponCompletion() {
        return isILMTest() == false;
    }

    /**
     * Tests are themselves responsible for cleaning up templates, which speeds up build.
     */
    @Override
    protected boolean preserveTemplatesUponCompletion() {
        return true;
    }

    protected boolean isSLMTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\") || (testName.contains("\\slm/")) ||
            // TODO: Remove after backport of https://github.com/elastic/elasticsearch/pull/48705 which moves SLM docs to correct folder
            testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
    }

    protected boolean isILMTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
    }

    protected boolean isMachineLearningTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/ml/") || testName.contains("\\ml\\"));
    }

    protected boolean isTransformTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/transform/") || testName.contains("\\transform\\"));
    }

    protected boolean isCcrTest() {
        String testName = getTestName();
        return testName != null && testName.contains("/ccr/");
    }

    protected boolean isGetLicenseTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/get-license/") || testName.contains("\\get-license\\"));
    }

    /**
     * Compares the results of running two analyzers against many random
     * strings. The goal is to figure out if two anlayzers are "the same" by
     * comparing their results. This is far from perfect but should be fairly
     * accurate, especially for gross things like missing {@code decimal_digit}
     * token filters, and should be fairly fast because it compares a fairly
     * small number of tokens.
     */
    private static class CompareAnalyzers implements ExecutableSection {
        private static final ConstructingObjectParser<CompareAnalyzers, XContentLocation> PARSER =
            new ConstructingObjectParser<>("test_analyzer", false, (a, location) -> {
                String index = (String) a[0];
                String first = (String) a[1];
                String second = (String) a[2];
                return new CompareAnalyzers(location, index, first, second);
            });
        static {
            PARSER.declareString(constructorArg(), new ParseField("index"));
            PARSER.declareString(constructorArg(), new ParseField("first"));
            PARSER.declareString(constructorArg(), new ParseField("second"));
        }
        private static CompareAnalyzers parse(XContentParser parser) throws IOException {
            XContentLocation location = parser.getTokenLocation();<commit_msg>Ensure license is ready for xpack info doc tests (#60706)<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.smoketest;

import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.http.HttpHost;
import org.apache.http.util.EntityUtils;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.Version;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ClientYamlDocsTestClient;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ClientYamlTestClient;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec;
import org.elasticsearch.test.rest.yaml.section.ExecutableSection;
import org.junit.After;
import org.junit.Before;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;

//The default 20 minutes timeout isn't always enough, please do not increase further than 30 before analyzing what makes this suite so slow
@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
public class DocsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {

    public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
        super(testCandidate);
    }

    @ParametersFactory
    public static Iterable<Object[]> parameters() throws Exception {
        List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS.size() + 1);
        entries.addAll(ExecutableSection.DEFAULT_EXECUTABLE_CONTEXTS);
        entries.add(new NamedXContentRegistry.Entry(ExecutableSection.class,
                new ParseField("compare_analyzers"), CompareAnalyzers::parse));
        NamedXContentRegistry executableSectionRegistry = new NamedXContentRegistry(entries);
        return ESClientYamlSuiteTestCase.createParameters(executableSectionRegistry);
    }

    @Override
    protected void afterIfFailed(List<Throwable> errors) {
        super.afterIfFailed(errors);
        String name = getTestName().split("=")[1];
        name = name.substring(0, name.length() - 1);
        name = name.replaceAll("/([^/]+)$", ".asciidoc:$1");
        logger.error("This failing test was generated by documentation starting at {}. It may include many snippets. "
                + "See docs/README.asciidoc for an explanation of test generation.", name);
    }

    @Override
    protected boolean randomizeContentType() {
        return false;
    }

    @Override
    protected ClientYamlTestClient initClientYamlTestClient(
            final ClientYamlSuiteRestSpec restSpec,
            final RestClient restClient,
            final List<HttpHost> hosts,
            final Version esVersion,
            final Version masterVersion) {
        return new ClientYamlDocsTestClient(restSpec, restClient, hosts, esVersion, masterVersion, this::getClientBuilderWithSniffedHosts);
    }

    @Before
    public void waitForRequirements() throws Exception {
        if (isCcrTest() || isGetLicenseTest() || isXpackInfoTest()) {
            ESRestTestCase.waitForActiveLicense(adminClient());
        }
    }

    @After
    public void cleanup() throws Exception {
        if (isMachineLearningTest() || isTransformTest()) {
            ESRestTestCase.waitForPendingTasks(adminClient());
        }

        // check that there are no templates
        Request request = new Request("GET", "_cat/templates");
        request.addParameter("h", "name");
        String templates = EntityUtils.toString(adminClient().performRequest(request).getEntity());
        if (false == "".equals(templates)) {
            for (String template : templates.split("\n")) {
                if (isXPackTemplate(template)) continue;
                if ("".equals(template)) {
                    throw new IllegalStateException("empty template in templates list:\n" + templates);
                }
                throw new RuntimeException("Template " + template + " not cleared after test");
            }
        }
    }

    @Override
    protected boolean preserveSLMPoliciesUponCompletion() {
        return isSLMTest() == false;
    }

    @Override
    protected boolean preserveILMPoliciesUponCompletion() {
        return isILMTest() == false;
    }

    /**
     * Tests are themselves responsible for cleaning up templates, which speeds up build.
     */
    @Override
    protected boolean preserveTemplatesUponCompletion() {
        return true;
    }

    protected boolean isSLMTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/slm/") || testName.contains("\\slm\\") || (testName.contains("\\slm/")) ||
            // TODO: Remove after backport of https://github.com/elastic/elasticsearch/pull/48705 which moves SLM docs to correct folder
            testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
    }

    protected boolean isILMTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/ilm/") || testName.contains("\\ilm\\") || testName.contains("\\ilm/"));
    }

    protected boolean isMachineLearningTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/ml/") || testName.contains("\\ml\\"));
    }

    protected boolean isTransformTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/transform/") || testName.contains("\\transform\\"));
    }

    protected boolean isCcrTest() {
        String testName = getTestName();
        return testName != null && testName.contains("/ccr/");
    }

    protected boolean isGetLicenseTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/get-license/") || testName.contains("\\get-license\\"));
    }

    protected boolean isXpackInfoTest() {
        String testName = getTestName();
        return testName != null && (testName.contains("/info/") || testName.contains("\\info\\"));
    }

    /**
     * Compares the results of running two analyzers against many random
     * strings. The goal is to figure out if two anlayzers are "the same" by
     * comparing their results. This is far from perfect but should be fairly
     * accurate, especially for gross things like missing {@code decimal_digit}
     * token filters, and should be fairly fast because it compares a fairly
     * small number of tokens.
     */
    private static class CompareAnalyzers implements ExecutableSection {
        private static final ConstructingObjectParser<CompareAnalyzers, XContentLocation> PARSER =
            new ConstructingObjectParser<>("test_analyzer", false, (a, location) -> {
                String index = (String) a[0];
                String first = (String) a[1];
                String second = (String) a[2];
                return new CompareAnalyzers(location, index, first, second);
            });
        static {
            PARSER.declareString(constructorArg(), new ParseField("index"));
            PARSER.declareString(constructorArg(), new ParseField("first"));
            PARSER.declareString(constructorArg(), new ParseField("second"));
        }
        private static CompareAnalyzers parse(XContentParser parser) throws IOException {
            XContentLocation location = parser.getTokenLocation();
{'fraction_non_alphanumeric': 0.10254172575542708, 'fraction_numerical': 0.002781717028471692, 'mean_word_length': 3.4741337237676917, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2564297', 'n_tokens_mistral': 4770, 'n_tokens_neox': 4700, 'n_words': 1250}