<commit_before>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.upgrades;

import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.Version;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.yaml.ObjectPath;
import org.elasticsearch.xpack.core.watcher.condition.AlwaysCondition;
import org.junit.Before;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.not;

public class WatchBackwardsCompatibilityIT extends AbstractUpgradeTestCase {

    private final StringEntity entity = new StringEntity(watchBuilder()
            .trigger(schedule(interval("5m")))
            .input(simpleInput())
            .condition(AlwaysCondition.INSTANCE)
            .addAction("_action1", loggingAction("{{ctx.watch_id}}"))
            .buildAsBytes(XContentType.JSON)
            .utf8ToString(),
            ContentType.APPLICATION_JSON);
    private Nodes nodes;

    @Before
    public void waitForSecuritySetup() throws Exception {

        String masterNode = null;
        String catNodesResponse = EntityUtils.toString(
                client().performRequest("GET", "/_cat/nodes?h=id,master").getEntity(),
                StandardCharsets.UTF_8
        );
        for (String line : catNodesResponse.split("\n")) {
            int indexOfStar = line.indexOf('*'); // * in the node's output denotes it is master
            if (indexOfStar != -1) {
                masterNode = line.substring(0, indexOfStar).trim();
                break;
            }
        }
        assertNotNull(masterNode);
        final String masterNodeId = masterNode;

        assertBusy(() -> {
            try {
                Response nodeDetailsResponse = client().performRequest("GET", "/_nodes");
                ObjectPath path = ObjectPath.createFromResponse(nodeDetailsResponse);
                Map<String, Object> nodes = path.evaluate("nodes");
                assertThat(nodes.size(), greaterThanOrEqualTo(2));
                String masterVersion = null;
                for (String key : nodes.keySet()) {
                    // get the ES version number master is on
                    if (key.startsWith(masterNodeId)) {
                        masterVersion = path.evaluate("nodes." + key + ".version");
                        break;
                    }
                }
                assertNotNull(masterVersion);
                final String masterTemplateVersion = masterVersion;

                Response response = client().performRequest("GET", "/_cluster/state/metadata");
                ObjectPath objectPath = ObjectPath.createFromResponse(response);
                final String mappingsPath = "metadata.templates.security-index-template.mappings";
                Map<String, Object> mappings = objectPath.evaluate(mappingsPath);
                assertNotNull(mappings);
                assertThat(mappings.size(), greaterThanOrEqualTo(1));
                for (String key : mappings.keySet()) {
                    String templateVersion = objectPath.evaluate(mappingsPath + "." + key + "" +
                            "._meta.security-version");
                    final Version mVersion = Version.fromString(masterTemplateVersion);
                    final Version tVersion = Version.fromString(templateVersion);
                    assertEquals(mVersion, tVersion);
                }
            } catch (Exception e) {
                throw new AssertionError("failed to get cluster state", e);
            }
        });

        nodes = buildNodeAndVersions();
        logger.info("Nodes in cluster before test: bwc [{}], new [{}], master [{}]", nodes.getBWCNodes(), nodes.getNewNodes(),
                nodes.getMaster());

        Map<String, String> params = Collections.singletonMap("error_trace", "true");
        executeAgainstMasterNode(client -> {
            // create a watch before each test, most of the time this is just overwriting...
            assertOK(client.performRequest("PUT", "/_xpack/watcher/watch/my-watch", params, entity));
            // just a check to see if we can execute a watch, purely optional
            if (randomBoolean()) {
                assertOK(client.performRequest("POST", "/_xpack/watcher/watch/my-watch/_execute", params,
                        new StringEntity("{ \"record_execution\" : true }", ContentType.APPLICATION_JSON)));
            }
            if (randomBoolean()) {
                Map<String, String> ignore404Params = MapBuilder.newMapBuilder(params).put("ignore", "404").immutableMap();
                Response indexExistsResponse = client.performRequest("HEAD", "/.triggered_watches", ignore404Params);
                if (indexExistsResponse.getStatusLine().getStatusCode() == 404) {
                    logger.info("Created triggered watches index to ensure it gets upgraded");
                    client.performRequest("PUT", "/.triggered_watches");
                }
            }
        });

        // helping debugging output
        executeAgainstMasterNode(client -> {
            Map<String, String> filterPathParams = MapBuilder.newMapBuilder(params)
                    .put("filter_path", "*.template,*.index_patterns").immutableMap();
            Response r = client.performRequest("GET", "_template/*watch*", filterPathParams);
            logger.info("existing watcher templates response [{}]", EntityUtils.toString(r.getEntity(), StandardCharsets.UTF_8));
        });

        // set logging to debug
//        executeAgainstMasterNode(client -> {
//            StringEntity entity = new StringEntity("{ \"transient\" : { \"logger.org.elasticsearch.xpack.watcher\" : \"TRACE\" } }",
//                    ContentType.APPLICATION_JSON);
//            Response response = client.performRequest("PUT", "_cluster/settings", params, entity);
//            logger.info("cluster update settings response [{}]", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
//        });
    }

    @Override
    protected boolean preserveIndicesUponCompletion() {
        return true;
    }

    @Override
    protected boolean preserveTemplatesUponCompletion() {
        return true;
    }

    @Override
    protected Settings restClientSettings() {
        String token = "Basic " + Base64.getEncoder()
                .encodeToString(("test_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8));
        return Settings.builder()
                .put(ThreadContext.PREFIX + ".Authorization", token)
                .build();
    }

    public void testDummy() {
        // Dummy test so the suite doesn't fail with a no executed tests error.
        // Remove when the awaits fixes below (#44814) are resolved
        assertTrue(true);
    }

    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/44814")
    public void testWatcherStats() throws Exception {<commit_msg>Mute WatchBackwardsCompatibilityIT suite<commit_after>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.upgrades;

import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.rest.yaml.ObjectPath;
import org.elasticsearch.xpack.core.watcher.condition.AlwaysCondition;
import org.junit.Before;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction;
import static org.elasticsearch.xpack.watcher.client.WatchSourceBuilders.watchBuilder;
import static org.elasticsearch.xpack.watcher.input.InputBuilders.simpleInput;
import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule;
import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.not;

@LuceneTestCase.AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/44814")
public class WatchBackwardsCompatibilityIT extends AbstractUpgradeTestCase {

    private final StringEntity entity = new StringEntity(watchBuilder()
            .trigger(schedule(interval("5m")))
            .input(simpleInput())
            .condition(AlwaysCondition.INSTANCE)
            .addAction("_action1", loggingAction("{{ctx.watch_id}}"))
            .buildAsBytes(XContentType.JSON)
            .utf8ToString(),
            ContentType.APPLICATION_JSON);
    private Nodes nodes;

    @Before
    public void waitForSecuritySetup() throws Exception {

        String masterNode = null;
        String catNodesResponse = EntityUtils.toString(
                client().performRequest("GET", "/_cat/nodes?h=id,master").getEntity(),
                StandardCharsets.UTF_8
        );
        for (String line : catNodesResponse.split("\n")) {
            int indexOfStar = line.indexOf('*'); // * in the node's output denotes it is master
            if (indexOfStar != -1) {
                masterNode = line.substring(0, indexOfStar).trim();
                break;
            }
        }
        assertNotNull(masterNode);
        final String masterNodeId = masterNode;

        assertBusy(() -> {
            try {
                Response nodeDetailsResponse = client().performRequest("GET", "/_nodes");
                ObjectPath path = ObjectPath.createFromResponse(nodeDetailsResponse);
                Map<String, Object> nodes = path.evaluate("nodes");
                assertThat(nodes.size(), greaterThanOrEqualTo(2));
                String masterVersion = null;
                for (String key : nodes.keySet()) {
                    // get the ES version number master is on
                    if (key.startsWith(masterNodeId)) {
                        masterVersion = path.evaluate("nodes." + key + ".version");
                        break;
                    }
                }
                assertNotNull(masterVersion);
                final String masterTemplateVersion = masterVersion;

                Response response = client().performRequest("GET", "/_cluster/state/metadata");
                ObjectPath objectPath = ObjectPath.createFromResponse(response);
                final String mappingsPath = "metadata.templates.security-index-template.mappings";
                Map<String, Object> mappings = objectPath.evaluate(mappingsPath);
                assertNotNull(mappings);
                assertThat(mappings.size(), greaterThanOrEqualTo(1));
                for (String key : mappings.keySet()) {
                    String templateVersion = objectPath.evaluate(mappingsPath + "." + key + "" +
                            "._meta.security-version");
                    final Version mVersion = Version.fromString(masterTemplateVersion);
                    final Version tVersion = Version.fromString(templateVersion);
                    assertEquals(mVersion, tVersion);
                }
            } catch (Exception e) {
                throw new AssertionError("failed to get cluster state", e);
            }
        });

        nodes = buildNodeAndVersions();
        logger.info("Nodes in cluster before test: bwc [{}], new [{}], master [{}]", nodes.getBWCNodes(), nodes.getNewNodes(),
                nodes.getMaster());

        Map<String, String> params = Collections.singletonMap("error_trace", "true");
        executeAgainstMasterNode(client -> {
            // create a watch before each test, most of the time this is just overwriting...
            assertOK(client.performRequest("PUT", "/_xpack/watcher/watch/my-watch", params, entity));
            // just a check to see if we can execute a watch, purely optional
            if (randomBoolean()) {
                assertOK(client.performRequest("POST", "/_xpack/watcher/watch/my-watch/_execute", params,
                        new StringEntity("{ \"record_execution\" : true }", ContentType.APPLICATION_JSON)));
            }
            if (randomBoolean()) {
                Map<String, String> ignore404Params = MapBuilder.newMapBuilder(params).put("ignore", "404").immutableMap();
                Response indexExistsResponse = client.performRequest("HEAD", "/.triggered_watches", ignore404Params);
                if (indexExistsResponse.getStatusLine().getStatusCode() == 404) {
                    logger.info("Created triggered watches index to ensure it gets upgraded");
                    client.performRequest("PUT", "/.triggered_watches");
                }
            }
        });

        // helping debugging output
        executeAgainstMasterNode(client -> {
            Map<String, String> filterPathParams = MapBuilder.newMapBuilder(params)
                    .put("filter_path", "*.template,*.index_patterns").immutableMap();
            Response r = client.performRequest("GET", "_template/*watch*", filterPathParams);
            logger.info("existing watcher templates response [{}]", EntityUtils.toString(r.getEntity(), StandardCharsets.UTF_8));
        });

        // set logging to debug
//        executeAgainstMasterNode(client -> {
//            StringEntity entity = new StringEntity("{ \"transient\" : { \"logger.org.elasticsearch.xpack.watcher\" : \"TRACE\" } }",
//                    ContentType.APPLICATION_JSON);
//            Response response = client.performRequest("PUT", "_cluster/settings", params, entity);
//            logger.info("cluster update settings response [{}]", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
//        });
    }

    @Override
    protected boolean preserveIndicesUponCompletion() {
        return true;
    }

    @Override
    protected boolean preserveTemplatesUponCompletion() {
        return true;
    }

    @Override
    protected Settings restClientSettings() {
        String token = "Basic " + Base64.getEncoder()
                .encodeToString(("test_user:x-pack-test-password").getBytes(StandardCharsets.UTF_8));
        return Settings.builder()
                .put(ThreadContext.PREFIX + ".Authorization", token)
                .build();
    }

    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/44814")
    public void testWatcherStats() throws Exception {
{'fraction_non_alphanumeric': 0.11033503018823251, 'fraction_numerical': 0.004380253344382621, 'mean_word_length': 2.6728260869565217, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6725998', 'n_tokens_mistral': 4324, 'n_tokens_neox': 4187, 'n_words': 905}