<commit_before>package org.talend.dataprep.dataset;

import static com.jayway.restassured.RestAssured.given;
import static com.jayway.restassured.RestAssured.when;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs;

import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.IntegrationTest;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.dataset.service.analysis.SynchronousDataSetAnalyzer;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepository;

import com.jayway.restassured.RestAssured;

/**
 * This test ensures the data set service behaves as stated in <a
 * href="https://jira.talendforge.org/browse/TDP-157">https://jira.talendforge.org/browse/TDP-157</a>.
 */
@RunWith(SpringJUnit4ClassRunner.class)
@SpringApplicationConfiguration(classes = Application.class)
@WebAppConfiguration
@IntegrationTest
public class DataSetImportTest {

    @Value("${local.server.port}")
    public int port;

    @Autowired
    DataSetMetadataRepository dataSetMetadataRepository;

    private String dataSetId;

    @BeforeClass
    public static void enter() {
        // Set pause in analysis
        System.setProperty("DataSetImportTest.PausedAnalyzer", "1"); //$NON-NLS-1$ //$NON-NLS-2$
    }

    @AfterClass
    public static void leave() {
        // Set pause in analysis
        System.setProperty("DataSetImportTest.PausedAnalyzer", "0"); //$NON-NLS-1$ //$NON-NLS-2$
    }

    @Before
    public void setUp() {
        RestAssured.port = port;
    }

    @After
    public void tearDown() throws Exception {
        dataSetMetadataRepository.clear();
    }

    @Test
    public void testNothing(){
    }

    /**
     * Test 'importing' status: the data set should remain in 'importing' state as long as create operation isn't
     * completed.
     */
    public void testImportStatus() throws Exception {
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (!dataSetMetadataRepository.list().iterator().hasNext()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        // Data set should show as importing
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // Asserts when import is done
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        assertThat(metadata.getLifecycle().importing(), is(false));
        assertThat(metadata.getLifecycle().schemaAnalyzed(), is(true));
        // TDP-283: Quality analysis should be synchronous
        assertThat(metadata.getLifecycle().qualityAnalyzed(), is(true));
    }

    /**
     * Test 'importing' status with list operation: data set in 'importing' mode should not appear in results of the
     * list operation.
     */
    public void testListImported() throws Exception {
        assertThat(dataSetMetadataRepository.size(), is(0));
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (!dataSetMetadataRepository.list().iterator().hasNext()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        // Find data set being imported...
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // ... list operation should *not* return data set being imported...
        when().get("/datasets").then().statusCode(HttpStatus.OK.value()).body(equalTo("[]"));
        // Assert the new data set is returned when creation completes.
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        assertThat(dataSetMetadataRepository.size(), is(1));
        String expected = "[{\"id\":\"" + metadata.getId() + "\"}]";
        when().get("/datasets").then().statusCode(HttpStatus.OK.value())
                .body(sameJSONAs(expected).allowingAnyArrayOrdering().allowingExtraUnexpectedFields());
    }

    /**
     * Test 'importing' status with get: user is not allowed to get data set content when it's still being imported. In
     * real life situation, this kind of event is rather unlikely since the UUID of the data set is only returned once
     * the creation completes (no access before this).
     */
    public void testCannotOpenDataSetBeingImported() throws Exception {
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (dataSetMetadataRepository.size() == 0) {
            TimeUnit.MILLISECONDS.sleep(50);
        }
        // Find data set being imported...
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // ... get operation should *not* return data set being imported but report an error ...
        int statusCode = when().get("/datasets/{id}/content", next.getId()).getStatusCode();
        assertThat(statusCode, is(400));
        // Assert the new data set is returned when creation completes.
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        statusCode = when().get("/datasets/{id}/content", metadata.getId()).getStatusCode();
        assertThat(statusCode, is(200));
    }

    /**
     * A special (for tests) implementation of {@link SynchronousDataSetAnalyzer} that allows test code to intentionally
     * slow down import process for test purposes.
     */
    @Component
    public static class PausedAnalyzer implements SynchronousDataSetAnalyzer {

        private static final Logger LOGGER = LoggerFactory.getLogger(PausedAnalyzer.class);

        @Override
        public int order() {
            return Integer.MAX_VALUE - 1;
        }

        @Override
        public void analyze(String dataSetId) {
            if (StringUtils.isEmpty(dataSetId)) {
                throw new IllegalArgumentException("Data set id cannot be null or empty.");
            }
            try {
                String timeToPause = System.getProperty("DataSetImportTest.PausedAnalyzer");
                if (!StringUtils.isEmpty(timeToPause)) {
                    LOGGER.info("Pausing import (for {} second(s))...", timeToPause);
                    TimeUnit.SECONDS.sleep(Integer.parseInt(timeToPause));
                    LOGGER.info("Pause done.");
                } else {
                    LOGGER.info("No pause.");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("Unable to pause import.", e);
            }
        }
    }
}
<commit_msg>TDP-880: enable failing unit tests<commit_after>package org.talend.dataprep.dataset;

import static com.jayway.restassured.RestAssured.given;
import static com.jayway.restassured.RestAssured.when;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.core.IsEqual.equalTo;
import static org.junit.Assert.assertThat;
import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs;

import java.io.IOException;
import java.util.Iterator;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.test.IntegrationTest;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.dataset.service.analysis.SynchronousDataSetAnalyzer;
import org.talend.dataprep.dataset.store.metadata.DataSetMetadataRepository;

import com.jayway.restassured.RestAssured;

/**
 * This test ensures the data set service behaves as stated in <a
 * href="https://jira.talendforge.org/browse/TDP-157">https://jira.talendforge.org/browse/TDP-157</a>.
 */
@RunWith(SpringJUnit4ClassRunner.class)
@SpringApplicationConfiguration(classes = Application.class)
@WebAppConfiguration
@IntegrationTest
public class DataSetImportTest {

    @Value("${local.server.port}")
    public int port;

    @Autowired
    DataSetMetadataRepository dataSetMetadataRepository;

    private String dataSetId;

    @BeforeClass
    public static void enter() {
        // Set pause in analysis
        System.setProperty("DataSetImportTest.PausedAnalyzer", "1"); //$NON-NLS-1$ //$NON-NLS-2$
    }

    @AfterClass
    public static void leave() {
        // Set pause in analysis
        System.setProperty("DataSetImportTest.PausedAnalyzer", "0"); //$NON-NLS-1$ //$NON-NLS-2$
    }

    @Before
    public void setUp() {
        RestAssured.port = port;
    }

    @After
    public void tearDown() throws Exception {
        dataSetMetadataRepository.clear();
    }

    /**
     * Test 'importing' status: the data set should remain in 'importing' state as long as create operation isn't
     * completed.
     */
    @Test
    public void testImportStatus() throws Exception {
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (!dataSetMetadataRepository.list().iterator().hasNext()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        // Data set should show as importing
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // Asserts when import is done
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        assertThat(metadata.getLifecycle().importing(), is(false));
        assertThat(metadata.getLifecycle().schemaAnalyzed(), is(true));
        // TDP-283: Quality analysis should be synchronous
        assertThat(metadata.getLifecycle().qualityAnalyzed(), is(true));
    }

    /**
     * Test 'importing' status with list operation: data set in 'importing' mode should not appear in results of the
     * list operation.
     */
    @Test
    public void testListImported() throws Exception {
        assertThat(dataSetMetadataRepository.size(), is(0));
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (!dataSetMetadataRepository.list().iterator().hasNext()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        // Find data set being imported...
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // ... list operation should *not* return data set being imported...
        when().get("/datasets").then().statusCode(HttpStatus.OK.value()).body(equalTo("[]"));
        // Assert the new data set is returned when creation completes.
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        assertThat(dataSetMetadataRepository.size(), is(1));
        String expected = "[{\"id\":\"" + metadata.getId() + "\"}]";
        when().get("/datasets").then().statusCode(HttpStatus.OK.value())
                .body(sameJSONAs(expected).allowingAnyArrayOrdering().allowingExtraUnexpectedFields());
    }

    /**
     * Test 'importing' status with get: user is not allowed to get data set content when it's still being imported. In
     * real life situation, this kind of event is rather unlikely since the UUID of the data set is only returned once
     * the creation completes (no access before this).
     */
    @Test
    public void testCannotOpenDataSetBeingImported() throws Exception {
        // Create a data set (asynchronously)
        Runnable creation = () -> {
            try {
                dataSetId = given().body(IOUtils.toString(DataSetImportTest.class.getResourceAsStream("tagada.csv")))
                        .queryParam("Content-Type", "text/csv").when().post("/datasets").asString();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        Thread creationThread = new Thread(creation);
        creationThread.start();
        // Wait for creation of data set object
        while (dataSetMetadataRepository.size() == 0) {
            TimeUnit.MILLISECONDS.sleep(50);
        }
        // Find data set being imported...
        final Iterable<DataSetMetadata> list = dataSetMetadataRepository.list();
        final Iterator<DataSetMetadata> iterator = list.iterator();
        assertThat(iterator.hasNext(), is(true));
        final DataSetMetadata next = iterator.next();
        assertThat(next.getLifecycle().importing(), is(true));
        // ... get operation should *not* return data set being imported but report an error ...
        int statusCode = when().get("/datasets/{id}/content", next.getId()).getStatusCode();
        assertThat(statusCode, is(400));
        // Assert the new data set is returned when creation completes.
        // Wait until creation is done (i.e. end of thread since creation is a blocking operation).
        creationThread.join();
        assertThat(dataSetId, notNullValue());
        final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId);
        statusCode = when().get("/datasets/{id}/content", metadata.getId()).getStatusCode();
        assertThat(statusCode, is(200));
    }

    /**
     * A special (for tests) implementation of {@link SynchronousDataSetAnalyzer} that allows test code to intentionally
     * slow down import process for test purposes.
     */
    @Component
    public static class PausedAnalyzer implements SynchronousDataSetAnalyzer {

        private static final Logger LOGGER = LoggerFactory.getLogger(PausedAnalyzer.class);

        @Override
        public int order() {
            return Integer.MAX_VALUE - 1;
        }

        @Override
        public void analyze(String dataSetId) {
            if (StringUtils.isEmpty(dataSetId)) {
                throw new IllegalArgumentException("Data set id cannot be null or empty.");
            }
            try {
                String timeToPause = System.getProperty("DataSetImportTest.PausedAnalyzer");
                if (!StringUtils.isEmpty(timeToPause)) {
                    LOGGER.info("Pausing import (for {} second(s))...", timeToPause);
                    TimeUnit.SECONDS.sleep(Integer.parseInt(timeToPause));
                    LOGGER.info("Pause done.");
                } else {
                    LOGGER.info("No pause.");
                }
            } catch (InterruptedException e) {
                throw new RuntimeException("Unable to pause import.", e);
            }
        }
    }
}

{'fraction_non_alphanumeric': 0.11372175742698262, 'fraction_numerical': 0.0037573267872351083, 'mean_word_length': 2.7943356776278274, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3439991', 'n_tokens_mistral': 5270, 'n_tokens_neox': 5226, 'n_words': 1277}