<commit_before>package com.firstblick.elasticsearch.module.export.test;

import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
import static com.github.tlrx.elasticsearch.test.EsSetup.deleteAll;
import static com.github.tlrx.elasticsearch.test.EsSetup.fromClassPath;

import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import junit.framework.TestCase;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.firstblick.elasticsearch.action.export.ExportAction;
import com.firstblick.elasticsearch.action.export.ExportRequest;
import com.firstblick.elasticsearch.action.export.ExportResponse;
import com.firstblick.elasticsearch.rest.action.admin.export.RestExportAction;
import com.github.tlrx.elasticsearch.test.EsSetup;

public class RestExportActionTest extends TestCase {

    ExportResponse response;
    CountDownLatch signal;
    EsSetup esSetup;
    RestExportAction restExportAction;

    /**
     * Action listener for export responses. Save response to member variable
     * and release the count down signal.
     */
    private class ExportResponseActionListener implements
            ActionListener<ExportResponse> {

        @Override
        public void onResponse(ExportResponse res) {
            response = res;
            signal.countDown();
        }

        @Override
        public void onFailure(Throwable e) {
            fail(e.getMessage());
        }

    }

    @Before
    public void setUp() {
        signal = new CountDownLatch(1);
        response = null;
        esSetup = new EsSetup();
        esSetup.execute(
                deleteAll(),
                createIndex("users")
                        .withSettings(
                                fromClassPath("essetup/settings/test_a.json"))
                        .withMapping("d",
                                fromClassPath("essetup/mappings/test_a.json"))
                        .withData(fromClassPath("essetup/data/test_a.json")));
        esSetup.client().admin().indices().prepareRefresh("users").execute();
    }

    @After
    public void tearDown() {
        esSetup.terminate();
    }

    /**
     * Either one of the parameters 'output_cmd' or 'output_file' is required.
     */
    @Test
    public void testNoCommandOrFile() {
        ExportRequest exportRequest = new ExportRequest();
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("'output_cmd' or 'output_file' has not been defined"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("'output_cmd' or 'output_file' has not been defined"));
    }

    /**
     * The parameter 'fields' is required.
     */
    @Test
    public void testNoExportFields() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest.source("{\"output_cmd\": \"cat\"}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("No export fields defined"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("No export fields defined"));
    }

    /**
     * Invalid parameters lead to an error response.
     */
    @Test
    public void testBadParserArgument() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": \"cat\", \"fields\": [\"name\"], \"badparam\":\"somevalue\"}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("No parser for element [badparam]"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("No parser for element [badparam]"));
    }

    /**
     * The 'output_cmd' parameter can be a single command and is executed.
     * The response shows the index, the executed command, the exit code of
     * the process and the process' standard out and standard error logs
     * (first 8K).
     */
    @Test
    public void testSingleOutputCommand() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": \"cat\", \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        // TODO adapt when working
        assertShardInfoCommand(infos.get(0), "users", -1, "", "Command failed",
                null);
        assertShardInfoCommand(infos.get(1), "users", -1, "", "Command failed",
                null);
    }

    /**
     * The 'output_cmd' parameter can also be a list of arguments.
     */
    @Test
    public void testOutputCommandList() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": [\"/bin/sh\", \"-c\", \"cat\"], \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        // TODO adapt when working
        assertShardInfoCommand(infos.get(0), "users", -1, "", "Command failed",
                null);
        assertShardInfoCommand(infos.get(1), "users", -1, "", "Command failed",
                null);
    }

    /**
     * The 'output_file' parameter defines the filename to save the export.
     * There are 3 template variables that will be replaced:
     *
     *  - ${cluster} : will be replaced with the cluster name
     *  - ${index}   : will be replaced with the index name
     *  - ${shard}   : will be replaced with the shard name
     *
     * The response contains the index, the shard number and the generated
     * output file name.
     */
    @Test
    public void testOutputFile() {
        String clusterName = esSetup.client().admin().cluster().prepareHealth().
                setWaitForGreenStatus().execute().actionGet().getClusterName();

        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_file\": \"/tmp/${cluster}.${shard}.${index}.export\", \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        Map<String, Object> shard_0 = infos.get(0);
        Map<String, Object> shard_1 = infos.get(1);
        assertEquals("users", shard_0.get("index"));
        assertEquals("users", shard_1.get("index"));<commit_msg>updated doc string<commit_after>package com.firstblick.elasticsearch.module.export.test;

import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
import static com.github.tlrx.elasticsearch.test.EsSetup.deleteAll;
import static com.github.tlrx.elasticsearch.test.EsSetup.fromClassPath;

import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;

import junit.framework.TestCase;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import com.firstblick.elasticsearch.action.export.ExportAction;
import com.firstblick.elasticsearch.action.export.ExportRequest;
import com.firstblick.elasticsearch.action.export.ExportResponse;
import com.firstblick.elasticsearch.rest.action.admin.export.RestExportAction;
import com.github.tlrx.elasticsearch.test.EsSetup;

public class RestExportActionTest extends TestCase {

    ExportResponse response;
    CountDownLatch signal;
    EsSetup esSetup;
    RestExportAction restExportAction;

    /**
     * Action listener for export responses. Save response to member variable
     * and release the count down signal.
     */
    private class ExportResponseActionListener implements
            ActionListener<ExportResponse> {

        @Override
        public void onResponse(ExportResponse res) {
            response = res;
            signal.countDown();
        }

        @Override
        public void onFailure(Throwable e) {
            fail(e.getMessage());
        }

    }

    @Before
    public void setUp() {
        signal = new CountDownLatch(1);
        response = null;
        esSetup = new EsSetup();
        esSetup.execute(
                deleteAll(),
                createIndex("users")
                        .withSettings(
                                fromClassPath("essetup/settings/test_a.json"))
                        .withMapping("d",
                                fromClassPath("essetup/mappings/test_a.json"))
                        .withData(fromClassPath("essetup/data/test_a.json")));
        esSetup.client().admin().indices().prepareRefresh("users").execute();
    }

    @After
    public void tearDown() {
        esSetup.terminate();
    }

    /**
     * Either one of the parameters 'output_cmd' or 'output_file' is required.
     */
    @Test
    public void testNoCommandOrFile() {
        ExportRequest exportRequest = new ExportRequest();
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("'output_cmd' or 'output_file' has not been defined"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("'output_cmd' or 'output_file' has not been defined"));
    }

    /**
     * The parameter 'fields' is required.
     */
    @Test
    public void testNoExportFields() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest.source("{\"output_cmd\": \"cat\"}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("No export fields defined"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("No export fields defined"));
    }

    /**
     * Invalid parameters lead to an error response.
     */
    @Test
    public void testBadParserArgument() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": \"cat\", \"fields\": [\"name\"], \"badparam\":\"somevalue\"}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        assertTrue(infos.get(0).get("error").toString()
                .contains("No parser for element [badparam]"));
        assertTrue(infos.get(1).get("error").toString()
                .contains("No parser for element [badparam]"));
    }

    /**
     * The 'output_cmd' parameter can be a single command and is executed.
     * The response shows the index, the node name, the shard number, the executed
     * command, the exit code of the process and the process' standard out and
     * standard error logs (first 8K) of every shard result.
     */
    @Test
    public void testSingleOutputCommand() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": \"cat\", \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        // TODO adapt when working
        assertShardInfoCommand(infos.get(0), "users", -1, "", "Command failed",
                null);
        assertShardInfoCommand(infos.get(1), "users", -1, "", "Command failed",
                null);
    }

    /**
     * The 'output_cmd' parameter can also be a list of arguments.
     */
    @Test
    public void testOutputCommandList() {
        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_cmd\": [\"/bin/sh\", \"-c\", \"cat\"], \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        // TODO adapt when working
        assertShardInfoCommand(infos.get(0), "users", -1, "", "Command failed",
                null);
        assertShardInfoCommand(infos.get(1), "users", -1, "", "Command failed",
                null);
    }

    /**
     * The 'output_file' parameter defines the filename to save the export.
     * There are 3 template variables that will be replaced:
     *
     *  - ${cluster} : will be replaced with the cluster name
     *  - ${index}   : will be replaced with the index name
     *  - ${shard}   : will be replaced with the shard name
     *
     * The response contains the index, the shard number, the node name and
     * the generated output file name of every shard result.
     */
    @Test
    public void testOutputFile() {
        String clusterName = esSetup.client().admin().cluster().prepareHealth().
                setWaitForGreenStatus().execute().actionGet().getClusterName();

        ExportRequest exportRequest = new ExportRequest();
        exportRequest
                .source("{\"output_file\": \"/tmp/${cluster}.${shard}.${index}.export\", \"fields\": [\"name\"]}");
        ActionListener<ExportResponse> listener = new ExportResponseActionListener();
        esSetup.client()
                .execute(ExportAction.INSTANCE, exportRequest, listener);

        waitForAsyncCallback(4000);

        List<Map<String, Object>> infos = response.getShardInfos();
        assertEquals(2, infos.size());
        Map<String, Object> shard_0 = infos.get(0);
        Map<String, Object> shard_1 = infos.get(1);
        assertEquals("users", shard_0.get("index"));
        assertEquals("users", shard_1.get("index"));
{'fraction_non_alphanumeric': 0.1237094945395954, 'fraction_numerical': 0.006325714626723161, 'mean_word_length': 2.6573548668703624, 'pattern_counts': {'":': 20, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6082402', 'n_tokens_mistral': 4507, 'n_tokens_neox': 4374, 'n_words': 1004}