<commit_before>package uk.ac.ebi.ddi.retriever;

import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.HttpClients;
import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.client.RestTemplate;
import uk.ac.ebi.ddi.extservices.utils.RetryClient;

import java.io.IOException;
import java.util.Collections;
import java.util.Set;

public abstract class DatasetFileUrlRetriever extends RetryClient implements IDatasetFileUrlRetriever {

    private IDatasetFileUrlRetriever datasetDownloadingRetriever;

    protected RestTemplate restTemplate;

    public DatasetFileUrlRetriever(IDatasetFileUrlRetriever datasetDownloadingRetriever) {
        this.datasetDownloadingRetriever = datasetDownloadingRetriever;
        this.restTemplate = new RestTemplate(clientHttpRequestFactory());
        MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter();
        converter.setSupportedMediaTypes(Collections.singletonList(MediaType.TEXT_HTML));
        restTemplate.getMessageConverters().add(converter);
    }

    protected abstract Set<String> getAllDatasetFiles(String accession, String database) throws IOException;

    @Override
    public Set<String> getDatasetFiles(String accession, String database) throws IOException {
        Set<String> result = datasetDownloadingRetriever.getDatasetFiles(accession, database);
        result.addAll(getAllDatasetFiles(accession, database));
        return result;
    }

    private ClientHttpRequestFactory clientHttpRequestFactory() {
        HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory();
        factory.setReadTimeout(200000);
        factory.setConnectTimeout(200000);
        HttpClient httpClient = HttpClients.custom()
                .setDefaultRequestConfig(RequestConfig.custom()
                        .setCookieSpec(CookieSpecs.STANDARD).build())
                .build();
        factory.setHttpClient(httpClient);
        return factory;
    }
}
<commit_msg>Ignore ssl verification<commit_after>package uk.ac.ebi.ddi.retriever;

import org.apache.http.client.HttpClient;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.HttpClients;
import org.springframework.http.MediaType;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.web.client.RestTemplate;
import uk.ac.ebi.ddi.extservices.utils.RetryClient;

import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.Collections;
import java.util.Set;

public abstract class DatasetFileUrlRetriever extends RetryClient implements IDatasetFileUrlRetriever {

    private IDatasetFileUrlRetriever datasetDownloadingRetriever;

    protected RestTemplate restTemplate;

    public DatasetFileUrlRetriever(IDatasetFileUrlRetriever datasetDownloadingRetriever) {
        this.datasetDownloadingRetriever = datasetDownloadingRetriever;
        try {
            this.restTemplate = new RestTemplate(clientHttpRequestFactory());
        } catch (KeyStoreException | NoSuchAlgorithmException | KeyManagementException e) {
            throw new RuntimeException(e);
        }
        MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter();
        converter.setSupportedMediaTypes(Collections.singletonList(MediaType.TEXT_HTML));
        restTemplate.getMessageConverters().add(converter);
    }

    protected abstract Set<String> getAllDatasetFiles(String accession, String database) throws IOException;

    @Override
    public Set<String> getDatasetFiles(String accession, String database) throws IOException {
        Set<String> result = datasetDownloadingRetriever.getDatasetFiles(accession, database);
        result.addAll(getAllDatasetFiles(accession, database));
        return result;
    }

    private ClientHttpRequestFactory clientHttpRequestFactory()
            throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException {
        HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory();
        TrustStrategy acceptingTrustStrategy = (X509Certificate[] chain, String authType) -> true;

        SSLContext sslContext = org.apache.http.ssl.SSLContexts.custom()
                .loadTrustMaterial(null, acceptingTrustStrategy)
                .build();

        SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
        factory.setReadTimeout(200000);
        factory.setConnectTimeout(200000);
        HttpClient httpClient = HttpClients.custom()
                .setSSLSocketFactory(csf)
                .setDefaultRequestConfig(RequestConfig.custom()
                        .setCookieSpec(CookieSpecs.STANDARD).build())
                .build();
        factory.setHttpClient(httpClient);
        return factory;
    }
}

{'fraction_non_alphanumeric': 0.07653150781112866, 'fraction_numerical': 0.00631911532385466, 'mean_word_length': 4.559024390243902, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6961141', 'n_tokens_mistral': 1486, 'n_tokens_neox': 1434, 'n_words': 247}