<commit_before>package org.talend.dataprep.dataset.store.content.http;

import java.io.IOException;
import java.io.InputStream;

import javax.annotation.PreDestroy;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.api.dataset.location.HttpLocation;
import org.talend.dataprep.dataset.store.content.DataSetContentStore;
import org.talend.dataprep.dataset.store.content.DataSetContentStoreAdapter;
import org.talend.dataprep.exception.TDPException;
import org.talend.dataprep.exception.error.DataSetErrorCodes;

/**
 * Remote http dataset content store implementation.
 */
@Component("ContentStore#http")
public class HttpContentStore extends DataSetContentStoreAdapter {

    /** This class' logger. */
    private static final Logger LOGGER = LoggerFactory.getLogger(HttpContentStore.class);

    /** Http connection manager. */
    private PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();

    /** The http client to use. */
    private CloseableHttpClient httpClient;

    /**
     * Default empty constructor.
     */
    public HttpContentStore() {
        connectionManager.setMaxTotal(50);
        connectionManager.setDefaultMaxPerRoute(50);
        httpClient = HttpClientBuilder.create() //
                .setConnectionManager(connectionManager) //
                .build();
    }

    /**
     * Clean the connection manager before shutting down.
     */
    @PreDestroy
    private void shutdown() {
        try {
            httpClient.close();
        } catch (IOException e) {
            LOGGER.error("Unable to close HTTP client on shutdown.", e);
        }
        this.connectionManager.shutdown();
    }

    /**
     * @see DataSetContentStore#getAsRaw(DataSetMetadata)
     */
    @Override
    public InputStream getAsRaw(DataSetMetadata dataSetMetadata) {
        HttpLocation location = (HttpLocation) dataSetMetadata.getLocation();
        HttpGet get = new HttpGet(location.getUrl());
        CloseableHttpResponse response;
        try {
            response = httpClient.execute(get);
            if (response.getStatusLine().getStatusCode() >= 400) {
                throw new IOException("error fetching " + location.getUrl() + " -> " + response.getStatusLine());
            }
            LOGGER.debug("HTTP remote dataset {} fetched from {}", dataSetMetadata, location.getUrl());
            return response.getEntity().getContent();
        } catch (IOException e) {
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_REMOTE_DATASET_CONTENT, e);
        }
    }

    /**
     * @see DataSetContentStore#storeAsRaw(DataSetMetadata, InputStream)
     */
    @Override
    public void storeAsRaw(DataSetMetadata dataSetMetadata, InputStream dataSetContent) {
        // nothing to do here since the dataset is already stored
    }

    /**
     * @see DataSetContentStore#delete(DataSetMetadata)
     */
    @Override
    public void delete(DataSetMetadata dataSetMetadata) {
        // nothing to do here<commit_msg>TDP-618: Add Accept to the header for HttpContentStore<commit_after>package org.talend.dataprep.dataset.store.content.http;

import java.io.IOException;
import java.io.InputStream;

import javax.annotation.PreDestroy;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.api.dataset.location.HttpLocation;
import org.talend.dataprep.dataset.store.content.DataSetContentStore;
import org.talend.dataprep.dataset.store.content.DataSetContentStoreAdapter;
import org.talend.dataprep.exception.TDPException;
import org.talend.dataprep.exception.error.DataSetErrorCodes;

/**
 * Remote http dataset content store implementation.
 */
@Component("ContentStore#http")
public class HttpContentStore extends DataSetContentStoreAdapter {

    /** This class' logger. */
    private static final Logger LOGGER = LoggerFactory.getLogger(HttpContentStore.class);

    /** Http connection manager. */
    private PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();

    /** The http client to use. */
    private CloseableHttpClient httpClient;

    /**
     * Default empty constructor.
     */
    public HttpContentStore() {
        connectionManager.setMaxTotal(50);
        connectionManager.setDefaultMaxPerRoute(50);
        httpClient = HttpClientBuilder.create() //
                .setConnectionManager(connectionManager) //
                .build();
    }

    /**
     * Clean the connection manager before shutting down.
     */
    @PreDestroy
    private void shutdown() {
        try {
            httpClient.close();
        } catch (IOException e) {
            LOGGER.error("Unable to close HTTP client on shutdown.", e);
        }
        this.connectionManager.shutdown();
    }

    /**
     * @see DataSetContentStore#getAsRaw(DataSetMetadata)
     */
    @Override
    public InputStream getAsRaw(DataSetMetadata dataSetMetadata) {
        HttpLocation location = (HttpLocation) dataSetMetadata.getLocation();
        HttpGet get = new HttpGet(location.getUrl());
        get.setHeader("Accept","*/*");
        CloseableHttpResponse response;
        try {
            response = httpClient.execute(get);
            if (response.getStatusLine().getStatusCode() >= 400) {
                throw new IOException("error fetching " + location.getUrl() + " -> " + response.getStatusLine());
            }
            LOGGER.debug("HTTP remote dataset {} fetched from {}", dataSetMetadata, location.getUrl());
            return response.getEntity().getContent();
        } catch (IOException e) {
            throw new TDPException(DataSetErrorCodes.UNABLE_TO_READ_REMOTE_DATASET_CONTENT, e);
        }
    }

    /**
     * @see DataSetContentStore#storeAsRaw(DataSetMetadata, InputStream)
     */
    @Override
    public void storeAsRaw(DataSetMetadata dataSetMetadata, InputStream dataSetContent) {
        // nothing to do here since the dataset is already stored
    }

    /**
     * @see DataSetContentStore#delete(DataSetMetadata)
     */
    @Override
    public void delete(DataSetMetadata dataSetMetadata) {
        // nothing to do here
{'fraction_non_alphanumeric': 0.10051694428489374, 'fraction_numerical': 0.0030155083285468122, 'mean_word_length': 3.273006134969325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3115471', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1862, 'n_words': 377}