<commit_before>package eu.stratuslab.storage.disk.utils;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.math.BigInteger;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.BasicClientConnectionManager;
import org.apache.http.protocol.HttpContext;

import eu.stratuslab.marketplace.metadata.MetadataUtils;

public class DownloadUtils {

    public static Map<String, BigInteger> copyUrlContentsToFile(String url,
            File file) throws IOException {

        Map<String, BigInteger> streamInfo = new HashMap<String, BigInteger>();

        DefaultHttpClient client = getHttpClient();

        try {

            HttpGet get = new HttpGet(url);

            HttpResponse response = client.execute(get);

            HttpEntity entity = response.getEntity();

            if (entity != null) {

                InputStream is = null;
                OutputStream os = null;

                try {
                    is = entity.getContent();
                    os = new FileOutputStream(file);
                    streamInfo = MetadataUtils.copyWithStreamInfo(is, os);

                } finally {
                    FileUtils.closeIgnoringError(is);
                    FileUtils.closeIgnoringError(os);
                }

                // Explicitly set the size of the output file.
                RandomAccessFile f = null;
                try {
                    f = new RandomAccessFile(file.getAbsolutePath(), "rw");
                    long length = streamInfo.get("BYTES").longValue();
                    f.setLength(length);
                } finally {
                    FileUtils.closeIgnoringError(f);
                }

            }

        } finally {
            client.getConnectionManager().shutdown();
        }

        return streamInfo;
    }

    private static class GzipRequestInterceptor implements
            HttpRequestInterceptor {

        public void process(final HttpRequest request, final HttpContext context)
                throws HttpException, IOException {

            if (!request.containsHeader("Accept-Encoding")) {
                request.addHeader("Accept-Encoding", "gzip");
            }
        }
    }

    private static class GzipResponseInterceptor implements
            HttpResponseInterceptor {

        public void process(final HttpResponse response,
                final HttpContext context) throws HttpException, IOException {

            HttpEntity entity = response.getEntity();
            if (entity != null) {
                Header ceheader = entity.getContentEncoding();
                if (ceheader != null) {
                    HeaderElement[] codecs = ceheader.getElements();
                    for (int i = 0; i < codecs.length; i++) {
                        if (codecs[i].getName().equalsIgnoreCase("gzip")) {
                            response.setEntity(new GzipDecompressingEntity(
                                    response.getEntity()));
                            return;
                        }
                    }
                }
            }
        }

    }

    private static DefaultHttpClient getHttpClient() {

        try {

            SSLContext sslContext = SSLContext.getInstance("SSL");

            sslContext.init(null, new TrustManager[] { new X509TrustManager() {

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[] {};
                }

                public void checkClientTrusted(X509Certificate[] certs,
                        String authType) {
                }

                public void checkServerTrusted(X509Certificate[] certs,
                        String authType) {
                }

            } }, new SecureRandom());

            SSLSocketFactory sf = new SSLSocketFactory(sslContext,
                    SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

            Scheme httpsScheme = new Scheme("https", 443, sf);
            SchemeRegistry schemeRegistry = new SchemeRegistry();
            schemeRegistry.register(httpsScheme);

            ClientConnectionManager cm = new BasicClientConnectionManager(
                    schemeRegistry);

            DefaultHttpClient client = new DefaultHttpClient(cm);

            client.addRequestInterceptor(new GzipRequestInterceptor());
            client.addResponseInterceptor(new GzipResponseInterceptor());

            return client;

        } catch (Exception e) {<commit_msg>ensure http is also a registered scheme for download<commit_after>package eu.stratuslab.storage.disk.utils;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.math.BigInteger;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;

import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.entity.GzipDecompressingEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.BasicClientConnectionManager;
import org.apache.http.protocol.HttpContext;

import eu.stratuslab.marketplace.metadata.MetadataUtils;

public class DownloadUtils {

    public static Map<String, BigInteger> copyUrlContentsToFile(String url,
            File file) throws IOException {

        Map<String, BigInteger> streamInfo = new HashMap<String, BigInteger>();

        DefaultHttpClient client = getHttpClient();

        try {

            HttpGet get = new HttpGet(url);

            HttpResponse response = client.execute(get);

            HttpEntity entity = response.getEntity();

            if (entity != null) {

                InputStream is = null;
                OutputStream os = null;

                try {
                    is = entity.getContent();
                    os = new FileOutputStream(file);
                    streamInfo = MetadataUtils.copyWithStreamInfo(is, os);

                } finally {
                    FileUtils.closeIgnoringError(is);
                    FileUtils.closeIgnoringError(os);
                }

                // Explicitly set the size of the output file.
                RandomAccessFile f = null;
                try {
                    f = new RandomAccessFile(file.getAbsolutePath(), "rw");
                    long length = streamInfo.get("BYTES").longValue();
                    f.setLength(length);
                } finally {
                    FileUtils.closeIgnoringError(f);
                }

            }

        } finally {
            client.getConnectionManager().shutdown();
        }

        return streamInfo;
    }

    private static class GzipRequestInterceptor implements
            HttpRequestInterceptor {

        public void process(final HttpRequest request, final HttpContext context)
                throws HttpException, IOException {

            if (!request.containsHeader("Accept-Encoding")) {
                request.addHeader("Accept-Encoding", "gzip");
            }
        }
    }

    private static class GzipResponseInterceptor implements
            HttpResponseInterceptor {

        public void process(final HttpResponse response,
                final HttpContext context) throws HttpException, IOException {

            HttpEntity entity = response.getEntity();
            if (entity != null) {
                Header ceheader = entity.getContentEncoding();
                if (ceheader != null) {
                    HeaderElement[] codecs = ceheader.getElements();
                    for (int i = 0; i < codecs.length; i++) {
                        if (codecs[i].getName().equalsIgnoreCase("gzip")) {
                            response.setEntity(new GzipDecompressingEntity(
                                    response.getEntity()));
                            return;
                        }
                    }
                }
            }
        }

    }

    private static DefaultHttpClient getHttpClient() {

        try {

            SSLContext sslContext = SSLContext.getInstance("SSL");

            sslContext.init(null, new TrustManager[] { new X509TrustManager() {

                public X509Certificate[] getAcceptedIssuers() {
                    return new X509Certificate[] {};
                }

                public void checkClientTrusted(X509Certificate[] certs,
                        String authType) {
                }

                public void checkServerTrusted(X509Certificate[] certs,
                        String authType) {
                }

            } }, new SecureRandom());

            SSLSocketFactory sf = new SSLSocketFactory(sslContext,
                    SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

            Scheme httpsScheme = new Scheme("https", 443, sf);
            Scheme httpScheme = new Scheme("http", 80, new PlainSocketFactory());
            SchemeRegistry schemeRegistry = new SchemeRegistry();
            schemeRegistry.register(httpsScheme);
            schemeRegistry.register(httpScheme);

            ClientConnectionManager cm = new BasicClientConnectionManager(
                    schemeRegistry);

            DefaultHttpClient client = new DefaultHttpClient(cm);

            client.addRequestInterceptor(new GzipRequestInterceptor());
            client.addResponseInterceptor(new GzipResponseInterceptor());

            return client;

        } catch (Exception e) {
{'fraction_non_alphanumeric': 0.08522066063915495, 'fraction_numerical': 0.004654910034911825, 'mean_word_length': 2.0358695652173915, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7154257', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2611, 'n_words': 576}