<commit_before>/*
 * Copyright (c) 2010-2011 Sonatype, Inc. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package com.ning.http.client.providers.jdk;

import com.ning.http.client.AsyncHandler;
import com.ning.http.client.AsyncHttpClientConfig;
import com.ning.http.client.AsyncHttpProvider;
import com.ning.http.client.AsyncHttpProviderConfig;
import com.ning.http.client.Body;
import com.ning.http.client.FluentCaseInsensitiveStringsMap;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.HttpResponseHeaders;
import com.ning.http.client.HttpResponseStatus;
import com.ning.http.client.ListenableFuture;
import com.ning.http.client.MaxRedirectException;
import com.ning.http.client.PerRequestConfig;
import com.ning.http.client.ProgressAsyncHandler;
import com.ning.http.client.ProxyServer;
import com.ning.http.client.Realm;
import com.ning.http.client.Request;
import com.ning.http.client.RequestBuilder;
import com.ning.http.client.Response;
import com.ning.http.client.filter.FilterContext;
import com.ning.http.client.filter.FilterException;
import com.ning.http.client.filter.IOExceptionFilter;
import com.ning.http.client.filter.ResponseFilter;
import com.ning.http.client.listener.TransferCompletionHandler;
import com.ning.http.multipart.MultipartRequestEntity;
import com.ning.http.util.AsyncHttpProviderUtils;
import com.ning.http.util.AuthenticatorUtils;
import com.ning.http.util.ProxyUtils;
import com.ning.http.util.SslUtils;
import com.ning.http.util.UTF8UrlEncoder;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.naming.AuthenticationException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSession;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.Authenticator;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.SocketAddress;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;

import static com.ning.http.util.AsyncHttpProviderUtils.DEFAULT_CHARSET;

public class JDKAsyncHttpProvider implements AsyncHttpProvider<HttpURLConnection> {
    private final static Logger logger = LoggerFactory.getLogger(JDKAsyncHttpProvider.class);

    private final static String NTLM_DOMAIN = "http.auth.ntlm.domain";

    private final AsyncHttpClientConfig config;

    private final AtomicBoolean isClose = new AtomicBoolean(false);

    private final static int MAX_BUFFERED_BYTES = 8192;

    private final AtomicInteger maxConnections = new AtomicInteger();

    private String jdkNtlmDomain;

    private Authenticator jdkAuthenticator;

    private boolean bufferResponseInMemory = false;

    public JDKAsyncHttpProvider(AsyncHttpClientConfig config) {

        this.config = config;
        AsyncHttpProviderConfig<?, ?> providerConfig = config.getAsyncHttpProviderConfig();
        if (providerConfig != null && JDKAsyncHttpProviderConfig.class.isAssignableFrom(providerConfig.getClass())) {
            configure(JDKAsyncHttpProviderConfig.class.cast(providerConfig));
        }
    }

    private void configure(JDKAsyncHttpProviderConfig config) {
        for(Map.Entry<String,String> e: config.propertiesSet()) {
            System.setProperty(e.getKey(), e.getValue());
        }

        if (config.getProperty(JDKAsyncHttpProviderConfig.FORCE_RESPONSE_BUFFERING) != null) {
            bufferResponseInMemory = true;
        }
    }
    public <T> ListenableFuture<T> execute(Request request, AsyncHandler<T> handler) throws IOException {
        return execute(request, handler, null);
    }
    public <T> ListenableFuture<T> execute(Request request, AsyncHandler<T> handler, ListenableFuture<?> future) throws IOException {
        if (isClose.get()) {
            throw new IOException("Closed");
        }

        if (config.getMaxTotalConnections() > -1 && (maxConnections.get() + 1) > config.getMaxTotalConnections()) {
            throw new IOException(String.format("Too many connections %s", config.getMaxTotalConnections()));
        }

        ProxyServer proxyServer = request.getProxyServer() != null ? request.getProxyServer() : config.getProxyServer();
        Realm realm =  request.getRealm() != null ?  request.getRealm() : config.getRealm();
        boolean avoidProxy = ProxyUtils.avoidProxy( proxyServer, request );
        Proxy proxy = null;
        if (!avoidProxy && (proxyServer != null || realm != null)) {
            try {
                proxy = configureProxyAndAuth(proxyServer, realm);
            } catch (AuthenticationException e) {
                throw new IOException(e.getMessage());
            }
        }

        HttpURLConnection urlConnection = createUrlConnection(request);

        PerRequestConfig conf = request.getPerRequestConfig();
        int requestTimeout = (conf != null && conf.getRequestTimeoutInMs() != 0) ?
                conf.getRequestTimeoutInMs() : config.getRequestTimeoutInMs();

        JDKDelegateFuture delegate = null;
        if (future != null) {
            delegate = new JDKDelegateFuture(handler, requestTimeout, future, urlConnection);
        }

        JDKFuture f = delegate == null ? new JDKFuture<T>(handler, requestTimeout, urlConnection) : delegate;
        f.touch();

        f.setInnerFuture(config.executorService().submit(new AsyncHttpUrlConnection(urlConnection, request, handler, f)));
        maxConnections.incrementAndGet();
        
        return f;
    }

    private HttpURLConnection createUrlConnection(Request request) throws IOException {
        ProxyServer proxyServer = request.getProxyServer() != null ? request.getProxyServer() : config.getProxyServer();
        Realm realm =  request.getRealm() != null ?  request.getRealm() : config.getRealm();
        boolean avoidProxy = ProxyUtils.avoidProxy( proxyServer, request );
        Proxy proxy = null;
        if (!avoidProxy && proxyServer != null || realm != null) {
            try {
                proxy = configureProxyAndAuth(proxyServer, realm);
            } catch (AuthenticationException e) {
                throw new IOException(e.getMessage());
            }
        }

        HttpURLConnection urlConnection = null;
        if (proxy == null) {
            urlConnection = (HttpURLConnection) AsyncHttpProviderUtils.createUri(request.getUrl()).toURL().openConnection();
        } else {
            urlConnection = (HttpURLConnection) AsyncHttpProviderUtils.createUri(request.getUrl()).toURL().openConnection(proxy);
        }
<commit_msg>o Do not pick up proxy properties by default in JDKAsyncHttpProvider (Part of AHC-75)<commit_after>/*
 * Copyright (c) 2010-2011 Sonatype, Inc. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package com.ning.http.client.providers.jdk;

import com.ning.http.client.AsyncHandler;
import com.ning.http.client.AsyncHttpClientConfig;
import com.ning.http.client.AsyncHttpProvider;
import com.ning.http.client.AsyncHttpProviderConfig;
import com.ning.http.client.Body;
import com.ning.http.client.FluentCaseInsensitiveStringsMap;
import com.ning.http.client.HttpResponseBodyPart;
import com.ning.http.client.HttpResponseHeaders;
import com.ning.http.client.HttpResponseStatus;
import com.ning.http.client.ListenableFuture;
import com.ning.http.client.MaxRedirectException;
import com.ning.http.client.PerRequestConfig;
import com.ning.http.client.ProgressAsyncHandler;
import com.ning.http.client.ProxyServer;
import com.ning.http.client.Realm;
import com.ning.http.client.Request;
import com.ning.http.client.RequestBuilder;
import com.ning.http.client.Response;
import com.ning.http.client.filter.FilterContext;
import com.ning.http.client.filter.FilterException;
import com.ning.http.client.filter.IOExceptionFilter;
import com.ning.http.client.filter.ResponseFilter;
import com.ning.http.client.listener.TransferCompletionHandler;
import com.ning.http.multipart.MultipartRequestEntity;
import com.ning.http.util.AsyncHttpProviderUtils;
import com.ning.http.util.AuthenticatorUtils;
import com.ning.http.util.ProxyUtils;
import com.ning.http.util.SslUtils;
import com.ning.http.util.UTF8UrlEncoder;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.naming.AuthenticationException;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSession;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.Authenticator;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.SocketAddress;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;

import static com.ning.http.util.AsyncHttpProviderUtils.DEFAULT_CHARSET;

public class JDKAsyncHttpProvider implements AsyncHttpProvider<HttpURLConnection> {
    private final static Logger logger = LoggerFactory.getLogger(JDKAsyncHttpProvider.class);

    private final static String NTLM_DOMAIN = "http.auth.ntlm.domain";

    private final AsyncHttpClientConfig config;

    private final AtomicBoolean isClose = new AtomicBoolean(false);

    private final static int MAX_BUFFERED_BYTES = 8192;

    private final AtomicInteger maxConnections = new AtomicInteger();

    private String jdkNtlmDomain;

    private Authenticator jdkAuthenticator;

    private boolean bufferResponseInMemory = false;

    public JDKAsyncHttpProvider(AsyncHttpClientConfig config) {

        this.config = config;
        AsyncHttpProviderConfig<?, ?> providerConfig = config.getAsyncHttpProviderConfig();
        if (providerConfig != null && JDKAsyncHttpProviderConfig.class.isAssignableFrom(providerConfig.getClass())) {
            configure(JDKAsyncHttpProviderConfig.class.cast(providerConfig));
        }
    }

    private void configure(JDKAsyncHttpProviderConfig config) {
        for(Map.Entry<String,String> e: config.propertiesSet()) {
            System.setProperty(e.getKey(), e.getValue());
        }

        if (config.getProperty(JDKAsyncHttpProviderConfig.FORCE_RESPONSE_BUFFERING) != null) {
            bufferResponseInMemory = true;
        }
    }
    public <T> ListenableFuture<T> execute(Request request, AsyncHandler<T> handler) throws IOException {
        return execute(request, handler, null);
    }
    public <T> ListenableFuture<T> execute(Request request, AsyncHandler<T> handler, ListenableFuture<?> future) throws IOException {
        if (isClose.get()) {
            throw new IOException("Closed");
        }

        if (config.getMaxTotalConnections() > -1 && (maxConnections.get() + 1) > config.getMaxTotalConnections()) {
            throw new IOException(String.format("Too many connections %s", config.getMaxTotalConnections()));
        }

        ProxyServer proxyServer = request.getProxyServer() != null ? request.getProxyServer() : config.getProxyServer();
        Realm realm =  request.getRealm() != null ?  request.getRealm() : config.getRealm();
        boolean avoidProxy = ProxyUtils.avoidProxy( proxyServer, request );
        Proxy proxy = null;
        if (!avoidProxy && (proxyServer != null || realm != null)) {
            try {
                proxy = configureProxyAndAuth(proxyServer, realm);
            } catch (AuthenticationException e) {
                throw new IOException(e.getMessage());
            }
        }

        HttpURLConnection urlConnection = createUrlConnection(request);

        PerRequestConfig conf = request.getPerRequestConfig();
        int requestTimeout = (conf != null && conf.getRequestTimeoutInMs() != 0) ?
                conf.getRequestTimeoutInMs() : config.getRequestTimeoutInMs();

        JDKDelegateFuture delegate = null;
        if (future != null) {
            delegate = new JDKDelegateFuture(handler, requestTimeout, future, urlConnection);
        }

        JDKFuture f = delegate == null ? new JDKFuture<T>(handler, requestTimeout, urlConnection) : delegate;
        f.touch();

        f.setInnerFuture(config.executorService().submit(new AsyncHttpUrlConnection(urlConnection, request, handler, f)));
        maxConnections.incrementAndGet();
        
        return f;
    }

    private HttpURLConnection createUrlConnection(Request request) throws IOException {
        ProxyServer proxyServer = request.getProxyServer() != null ? request.getProxyServer() : config.getProxyServer();
        Realm realm =  request.getRealm() != null ?  request.getRealm() : config.getRealm();
        boolean avoidProxy = ProxyUtils.avoidProxy( proxyServer, request );
        Proxy proxy = null;
        if (!avoidProxy && proxyServer != null || realm != null) {
            try {
                proxy = configureProxyAndAuth(proxyServer, realm);
            } catch (AuthenticationException e) {
                throw new IOException(e.getMessage());
            }
        }

        HttpURLConnection urlConnection = null;
        if (proxy == null) {
            urlConnection =
                (HttpURLConnection) AsyncHttpProviderUtils.createUri( request.getUrl() ).toURL().openConnection( Proxy.NO_PROXY );
        } else {
            urlConnection = (HttpURLConnection) AsyncHttpProviderUtils.createUri(request.getUrl()).toURL().openConnection(proxy);
        }

{'fraction_non_alphanumeric': 0.09842177208897726, 'fraction_numerical': 0.003852367341866534, 'mean_word_length': 3.9584103512014788, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5109253', 'n_tokens_mistral': 4427, 'n_tokens_neox': 4299, 'n_words': 1004}