<commit_before>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.watcher.actions.index;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.watcher.actions.Action;
import org.elasticsearch.watcher.actions.Action.Result.Status;
import org.elasticsearch.watcher.execution.WatchExecutionContext;
import org.elasticsearch.watcher.support.WatcherDateTimeUtils;
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
import org.elasticsearch.watcher.support.xcontent.XContentSource;
import org.elasticsearch.watcher.test.WatcherTestUtils;
import org.elasticsearch.watcher.watch.Payload;
import org.joda.time.DateTime;

import java.util.Arrays;
import java.util.Map;

import static java.util.Collections.singletonMap;
import static java.util.Collections.unmodifiableSet;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.joda.time.DateTimeZone.UTC;

/**
 */
public class IndexActionTests extends ESIntegTestCase {
    @Override
    protected Settings nodeSettings(int nodeOrdinal) {
        return Settings.builder()
                .put(super.nodeSettings(nodeOrdinal))
                .build();
    }

    @Override
    protected Settings transportClientSettings() {
        return Settings.builder()
                .put(super.transportClientSettings())
                .build();
    }

    public void testIndexActionExecuteSingleDoc() throws Exception {
        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
        boolean customTimestampField = "@timestamp".equals(timestampField);

        if (timestampField == null || "_timestamp".equals(timestampField)) {
            assertThat(prepareCreate("test-index")
                    .addMapping("test-type", "{ \"test-type\" : { \"_timestamp\" : { \"enabled\" : \"true\" }}}")
                    .get().isAcknowledged(), is(true));
        }

        IndexAction action = new IndexAction("test-index", "test-type", timestampField, null, null);
        ExecutableIndexAction executable = new ExecutableIndexAction(action, logger, WatcherClientProxy.of(client()), null);
        DateTime executionTime = DateTime.now(UTC);
        Payload payload = randomBoolean() ? new Payload.Simple("foo", "bar") : new Payload.Simple("_doc", singletonMap("foo", "bar"));
        WatchExecutionContext ctx = WatcherTestUtils.mockExecutionContext("_id", executionTime, payload);

        Action.Result result = executable.execute("_id", ctx, ctx.payload());

        assertThat(result.status(), equalTo(Status.SUCCESS));
        assertThat(result, instanceOf(IndexAction.Result.Success.class));
        IndexAction.Result.Success successResult = (IndexAction.Result.Success) result;
        XContentSource response = successResult.response();
        assertThat(response.getValue("created"), equalTo((Object)Boolean.TRUE));
        assertThat(response.getValue("version"), equalTo((Object) 1));
        assertThat(response.getValue("type").toString(), equalTo("test-type"));
        assertThat(response.getValue("index").toString(), equalTo("test-index"));

        refresh(); //Manually refresh to make sure data is available

        SearchResponse searchResponse = client().prepareSearch("test-index")
                .setTypes("test-type")
                .setSource(searchSource()
                        .query(matchAllQuery())
                        .aggregation(terms("timestamps").field(customTimestampField ? timestampField : "_timestamp")))
                .get();

        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
        SearchHit hit = searchResponse.getHits().getAt(0);

        if (customTimestampField) {
            assertThat(hit.getSource().size(), is(2));
            assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
            assertThat(hit.getSource(), hasEntry(timestampField, (Object) WatcherDateTimeUtils.formatDate(executionTime)));
        } else {
            assertThat(hit.getSource().size(), is(1));
            assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
        }
        Terms terms = searchResponse.getAggregations().get("timestamps");
        assertThat(terms, notNullValue());
        assertThat(terms.getBuckets(), hasSize(1));
        assertThat(terms.getBuckets().get(0).getKeyAsNumber().longValue(), is(executionTime.getMillis()));
        assertThat(terms.getBuckets().get(0).getDocCount(), is(1L));
    }

    public void testIndexActionExecuteMultiDoc() throws Exception {
        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
        boolean customTimestampField = "@timestamp".equals(timestampField);

        if (timestampField == null || "_timestamp".equals(timestampField)) {
            assertThat(prepareCreate("test-index")
                    .addMapping("test-type", "_timestamp", "enabled=true", "foo", "type=keyword")
                    .get().isAcknowledged(), is(true));
        }

        Object list = randomFrom(
                new Map[] { singletonMap("foo", "bar"), singletonMap("foo", "bar1") },
                Arrays.asList(singletonMap("foo", "bar"), singletonMap("foo", "bar1")),
                unmodifiableSet(newHashSet(singletonMap("foo", "bar"), singletonMap("foo", "bar1")))
        );

        IndexAction action = new IndexAction("test-index", "test-type", timestampField, null, null);
        ExecutableIndexAction executable = new ExecutableIndexAction(action, logger, WatcherClientProxy.of(client()), null);
        DateTime executionTime = DateTime.now(UTC);
        WatchExecutionContext ctx = WatcherTestUtils.mockExecutionContext("_id", executionTime, new Payload.Simple("_doc", list));

        Action.Result result = executable.execute("_id", ctx, ctx.payload());

        assertThat(result.status(), equalTo(Status.SUCCESS));
        assertThat(result, instanceOf(IndexAction.Result.Success.class));
        IndexAction.Result.Success successResult = (IndexAction.Result.Success) result;
        XContentSource response = successResult.response();
        assertThat(successResult.toString(), response.getValue("0.created"), equalTo((Object)Boolean.TRUE));
        assertThat(successResult.toString(), response.getValue("0.version"), equalTo((Object) 1));
        assertThat(successResult.toString(), response.getValue("0.type").toString(), equalTo("test-type"));
        assertThat(successResult.toString(), response.getValue("0.index").toString(), equalTo("test-index"));
        assertThat(successResult.toString(), response.getValue("1.created"), equalTo((Object)Boolean.TRUE));<commit_msg>IndexActionTests: Always map `foo` as a keyword.<commit_after>/*
 * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
 * or more contributor license agreements. Licensed under the Elastic License;
 * you may not use this file except in compliance with the Elastic License.
 */
package org.elasticsearch.watcher.actions.index;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.watcher.actions.Action;
import org.elasticsearch.watcher.actions.Action.Result.Status;
import org.elasticsearch.watcher.execution.WatchExecutionContext;
import org.elasticsearch.watcher.support.WatcherDateTimeUtils;
import org.elasticsearch.watcher.support.init.proxy.WatcherClientProxy;
import org.elasticsearch.watcher.support.xcontent.XContentSource;
import org.elasticsearch.watcher.test.WatcherTestUtils;
import org.elasticsearch.watcher.watch.Payload;
import org.joda.time.DateTime;

import java.util.Arrays;
import java.util.Map;

import static java.util.Collections.singletonMap;
import static java.util.Collections.unmodifiableSet;
import static org.elasticsearch.common.util.set.Sets.newHashSet;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.joda.time.DateTimeZone.UTC;

/**
 */
public class IndexActionTests extends ESIntegTestCase {
    @Override
    protected Settings nodeSettings(int nodeOrdinal) {
        return Settings.builder()
                .put(super.nodeSettings(nodeOrdinal))
                .build();
    }

    @Override
    protected Settings transportClientSettings() {
        return Settings.builder()
                .put(super.transportClientSettings())
                .build();
    }

    public void testIndexActionExecuteSingleDoc() throws Exception {
        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
        boolean customTimestampField = "@timestamp".equals(timestampField);

        if (timestampField == null || "_timestamp".equals(timestampField)) {
            assertThat(prepareCreate("test-index")
                    .addMapping("test-type", "{ \"test-type\" : { \"_timestamp\" : { \"enabled\" : \"true\" }}}")
                    .get().isAcknowledged(), is(true));
        }

        IndexAction action = new IndexAction("test-index", "test-type", timestampField, null, null);
        ExecutableIndexAction executable = new ExecutableIndexAction(action, logger, WatcherClientProxy.of(client()), null);
        DateTime executionTime = DateTime.now(UTC);
        Payload payload = randomBoolean() ? new Payload.Simple("foo", "bar") : new Payload.Simple("_doc", singletonMap("foo", "bar"));
        WatchExecutionContext ctx = WatcherTestUtils.mockExecutionContext("_id", executionTime, payload);

        Action.Result result = executable.execute("_id", ctx, ctx.payload());

        assertThat(result.status(), equalTo(Status.SUCCESS));
        assertThat(result, instanceOf(IndexAction.Result.Success.class));
        IndexAction.Result.Success successResult = (IndexAction.Result.Success) result;
        XContentSource response = successResult.response();
        assertThat(response.getValue("created"), equalTo((Object)Boolean.TRUE));
        assertThat(response.getValue("version"), equalTo((Object) 1));
        assertThat(response.getValue("type").toString(), equalTo("test-type"));
        assertThat(response.getValue("index").toString(), equalTo("test-index"));

        refresh(); //Manually refresh to make sure data is available

        SearchResponse searchResponse = client().prepareSearch("test-index")
                .setTypes("test-type")
                .setSource(searchSource()
                        .query(matchAllQuery())
                        .aggregation(terms("timestamps").field(customTimestampField ? timestampField : "_timestamp")))
                .get();

        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
        SearchHit hit = searchResponse.getHits().getAt(0);

        if (customTimestampField) {
            assertThat(hit.getSource().size(), is(2));
            assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
            assertThat(hit.getSource(), hasEntry(timestampField, (Object) WatcherDateTimeUtils.formatDate(executionTime)));
        } else {
            assertThat(hit.getSource().size(), is(1));
            assertThat(hit.getSource(), hasEntry("foo", (Object) "bar"));
        }
        Terms terms = searchResponse.getAggregations().get("timestamps");
        assertThat(terms, notNullValue());
        assertThat(terms.getBuckets(), hasSize(1));
        assertThat(terms.getBuckets().get(0).getKeyAsNumber().longValue(), is(executionTime.getMillis()));
        assertThat(terms.getBuckets().get(0).getDocCount(), is(1L));
    }

    public void testIndexActionExecuteMultiDoc() throws Exception {
        String timestampField = randomFrom(null, "_timestamp", "@timestamp");
        boolean customTimestampField = "@timestamp".equals(timestampField);

        if (timestampField == null || "_timestamp".equals(timestampField)) {
            assertAcked(prepareCreate("test-index")
                    .addMapping("test-type", "_timestamp", "enabled=true", "foo", "type=keyword"));
        } else {
            assertAcked(prepareCreate("test-index")
                    .addMapping("test-type", "foo", "type=keyword"));
        }

        Object list = randomFrom(
                new Map[] { singletonMap("foo", "bar"), singletonMap("foo", "bar1") },
                Arrays.asList(singletonMap("foo", "bar"), singletonMap("foo", "bar1")),
                unmodifiableSet(newHashSet(singletonMap("foo", "bar"), singletonMap("foo", "bar1")))
        );

        IndexAction action = new IndexAction("test-index", "test-type", timestampField, null, null);
        ExecutableIndexAction executable = new ExecutableIndexAction(action, logger, WatcherClientProxy.of(client()), null);
        DateTime executionTime = DateTime.now(UTC);
        WatchExecutionContext ctx = WatcherTestUtils.mockExecutionContext("_id", executionTime, new Payload.Simple("_doc", list));

        Action.Result result = executable.execute("_id", ctx, ctx.payload());

        assertThat(result.status(), equalTo(Status.SUCCESS));
        assertThat(result, instanceOf(IndexAction.Result.Success.class));
        IndexAction.Result.Success successResult = (IndexAction.Result.Success) result;
        XContentSource response = successResult.response();
        assertThat(successResult.toString(), response.getValue("0.created"), equalTo((Object)Boolean.TRUE));
        assertThat(successResult.toString(), response.getValue("0.version"), equalTo((Object) 1));
        assertThat(successResult.toString(), response.getValue("0.type").toString(), equalTo("test-type"));
        assertThat(successResult.toString(), response.getValue("0.index").toString(), equalTo("test-index"));
        assertThat(successResult.toString(), response.getValue("1.created"), equalTo((Object)Boolean.TRUE));
{'fraction_non_alphanumeric': 0.13021287190339847, 'fraction_numerical': 0.00224075687787875, 'mean_word_length': 4.011540860885839, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7493315', 'n_tokens_mistral': 4166, 'n_tokens_neox': 4164, 'n_words': 771}