<commit_before>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.smoketest;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.test.rest.ESRestTestCase;

import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;

import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.containsString;

public class ReindexFromOldRemoteIT extends ESRestTestCase {
    private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
        int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
        try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
            try {
                HttpEntity entity = new StringEntity("{\"settings\":{\"number_of_shards\": 1}}", ContentType.APPLICATION_JSON);
                oldEs.performRequest("PUT", "/test", singletonMap("refresh", "true"), entity);

                entity = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON);
                oldEs.performRequest("PUT", "/test/doc/testdoc1", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc2", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc3", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc4", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc5", singletonMap("refresh", "true"), entity);

                entity = new StringEntity(
                        "{\n"
                      + "  \"source\":{\n"
                      + "    \"index\": \"test\",\n"
                      + "    \"size\": 1,\n"
                      + "    \"remote\": {\n"
                      + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
                      + "    }\n"
                      + "  },\n"
                      + "  \"dest\": {\n"
                      + "    \"index\": \"test\"\n"
                      + "  }\n"
                      + "}",
                      ContentType.APPLICATION_JSON);
                Map<String, String> params = new TreeMap<>();
                params.put("refresh", "true");
                params.put("pretty", "true");
                if (requestsPerSecond != null) {
                    params.put("requests_per_second", requestsPerSecond);
                }
                client().performRequest("POST", "/_reindex", params, entity);

                Response response = client().performRequest("POST", "test/_search", singletonMap("pretty", "true"));
                String result = EntityUtils.toString(response.getEntity());
                assertThat(result, containsString("\"_id\" : \"testdoc1\""));
            } finally {
                oldEs.performRequest("DELETE", "/test");
            }
        }
    }

    public void testEs2() throws IOException {
        oldEsTestCase("es2.port", null);
    }

    public void testEs1() throws IOException {
        oldEsTestCase("es1.port", null);
    }

    public void testEs090() throws IOException {
        oldEsTestCase("es090.port", null);
    }

    public void testEs2WithFunnyThrottle() throws IOException {
        oldEsTestCase("es2.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
    }

    public void testEs1WithFunnyThrottle() throws IOException {
        oldEsTestCase("es1.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
    }<commit_msg>Reindex: Harden old ES version test<commit_after>/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.smoketest;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.test.rest.ESRestTestCase;

import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;

import static java.util.Collections.singletonMap;
import static org.hamcrest.Matchers.containsString;

public class ReindexFromOldRemoteIT extends ESRestTestCase {
    private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
        int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
        try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
            try {
                HttpEntity entity = new StringEntity("{\"settings\":{\"number_of_shards\": 1}}", ContentType.APPLICATION_JSON);
                oldEs.performRequest("PUT", "/test", singletonMap("refresh", "true"), entity);

                entity = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON);
                oldEs.performRequest("PUT", "/test/doc/testdoc1", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc2", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc3", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc4", singletonMap("refresh", "true"), entity);
                oldEs.performRequest("PUT", "/test/doc/testdoc5", singletonMap("refresh", "true"), entity);

                entity = new StringEntity(
                        "{\n"
                      + "  \"source\":{\n"
                      + "    \"index\": \"test\",\n"
                      + "    \"size\": 1,\n"
                      + "    \"remote\": {\n"
                      + "      \"host\": \"http://127.0.0.1:" + oldEsPort + "\"\n"
                      + "    }\n"
                      + "  },\n"
                      + "  \"dest\": {\n"
                      + "    \"index\": \"test\"\n"
                      + "  }\n"
                      + "}",
                      ContentType.APPLICATION_JSON);
                Map<String, String> params = new TreeMap<>();
                params.put("refresh", "true");
                params.put("pretty", "true");
                if (requestsPerSecond != null) {
                    params.put("requests_per_second", requestsPerSecond);
                }
                client().performRequest("POST", "/_reindex", params, entity);

                Response response = client().performRequest("POST", "test/_search", singletonMap("pretty", "true"));
                String result = EntityUtils.toString(response.getEntity());
                assertThat(result, containsString("\"_id\" : \"testdoc1\""));
            } finally {
                try {
                    oldEs.performRequest("DELETE", "/test");
                } catch (ResponseException e) {
                    /* Try not to throw ResponseException for as it'll eat the
                     * real exception. This is because the rest client throws
                     * exceptions in a "funny" way that isn't compatible with
                     * `suppressed`. In the case of 404s we'll just log something
                     * and move on because that just means that a previous
                     * failure caused the index not to be created. */
                    if (e.getResponse().getStatusLine().getStatusCode() == 404) {
                        logger.warn("old index not deleted because it doesn't exist");
                    } else {
                        logger.error("failed to remove old index", e);
                        fail("failed to remove old index, see log");
                    }
                }
            }
        }
    }

    public void testEs2() throws IOException {
        oldEsTestCase("es2.port", null);
    }

    public void testEs1() throws IOException {
        oldEsTestCase("es1.port", null);
    }

    public void testEs090() throws IOException {
        oldEsTestCase("es090.port", null);
    }

    public void testEs2WithFunnyThrottle() throws IOException {
        oldEsTestCase("es2.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
    }

    public void testEs1WithFunnyThrottle() throws IOException {
        oldEsTestCase("es1.port", "11"); // 11 requests per second should give us a nice "funny" number on the scroll timeout
    }
{'fraction_non_alphanumeric': 0.12416140009722898, 'fraction_numerical': 0.009528439474963539, 'mean_word_length': 2.2417270721714466, 'pattern_counts': {'":': 20, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6444582', 'n_tokens_mistral': 2723, 'n_tokens_neox': 2576, 'n_words': 736}