<commit_before>/*
 * Copyright 2015 JKOOL, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.jkool.jesl.net.http;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.ProtocolVersion;
import org.apache.http.conn.ClientConnectionRequest;
import org.apache.http.conn.ManagedClientConnection;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.routing.RouteInfo;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.conn.BasicClientConnectionManager;
import org.apache.http.params.BasicHttpParams;

import com.jkool.jesl.net.http.apache.HttpRequestImpl;
import com.jkool.jesl.net.http.apache.HttpResponseImpl;
import com.jkool.jesl.net.security.AccessResponse;
import com.jkool.jesl.net.security.AuthUtils;
import com.jkool.jesl.net.ssl.SSLContextFactory;
import com.nastel.jkool.tnt4j.core.OpLevel;
import com.nastel.jkool.tnt4j.sink.DefaultEventSinkFactory;
import com.nastel.jkool.tnt4j.sink.EventSink;
import com.nastel.jkool.tnt4j.utils.Utils;

/**
 * This class provides HTTP[S] connection to the specified JESK server based on given URL.
 *
 * @version $Revision: 3 $
 */
public class HttpClient implements HttpStream {
	public static final String NO_RESPONSE_REQUIRED_HEADER = "Pragma";
	public static final String NO_RESPONSE_REQUIRED_VALUE  = "no-response";

	protected URI							uri;
	protected EventSink		 			    logger;
	protected String						host;
	protected int							port;
	protected String						uriPath;
	protected String						sslKeystore;
	protected String						sslKeystorePwd;
	protected HttpHost						httpHost;
	protected HttpHost						httpProxy;
	protected BasicClientConnectionManager	connMgr;
	protected ClientConnectionRequest		connReq;
	protected ManagedClientConnection		connection;
	protected HttpRoute						route;
	protected SchemeRegistry				schemeReg;
	protected boolean						secure = false;

	/**
	 * Create JESL HTTP[S} client stream with given attributes
	 *
	 * @param urlStr connection string to specified JESL server
	 * @param logger event sink used for logging, null if none
	 * @throws URISyntaxException if invalid connection string
	 */
	public HttpClient(String urlStr, EventSink logger) throws URISyntaxException {
		this(urlStr, null, 0, logger);
	}

	/**
	 * Create JESL HTTP[S} client stream with given attributes
	 *
	 * @param urlStr connection string to specified JESL server
	 * @param proxyHost proxy host name if any, null if none
	 * @param proxyPort proxy port number if any, 0 of none
	 * @param logger event sink used for logging, null if none
	 * @throws URISyntaxException if invalid connection string
	 */
	public HttpClient(String urlStr, String proxyHost, int proxyPort, EventSink logger) throws URISyntaxException {
		uri = new URI(urlStr);
		String scheme = uri.getScheme();
		secure = "https".equalsIgnoreCase(scheme);
		host = uri.getHost();
	    if (host == null)
	    	host = "localhost";
	    port = uri.getPort();
	    if (port <= 0)
	    	port = (secure ? 443 : 80);
		init(host, port, uri.getPath(), secure, proxyHost, proxyPort, logger);
	}

	/**
	 *
	 */
	private void init(String host, int port, String uriPath, boolean secure,
					  String proxyHost, int proxyPort, EventSink logger) {
		this.host    = host;
		this.port    = port;
		this.uriPath = uriPath;
		this.secure  = secure;
		this.logger  = (logger != null ? logger : DefaultEventSinkFactory.defaultEventSink(HttpClient.class));

		String scheme = (secure ? "https" : "http");

	    httpHost = new HttpHost(host, port, scheme);
		if (!StringUtils.isEmpty(proxyHost))
			httpProxy = new HttpHost(proxyHost, proxyPort);
	}

	/**
	 * Sets keystore and password for obtaining SSL certificate for server.
	 *
	 * @param sslKeystore path to keystore
	 * @param sslKeystorePwd password for keystore
	 */
	public void setSslKeystore(String sslKeystore, String sslKeystorePwd) {
		this.sslKeystore    = sslKeystore;
		this.sslKeystorePwd = sslKeystorePwd;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void connect() throws IOException {
		try {
		    if (secure) {
		    	SSLSocketFactory ssf = null;
		    	if (!StringUtils.isEmpty(sslKeystore)) {
			    	SSLContextFactory scf = new SSLContextFactory(sslKeystore, sslKeystorePwd, sslKeystorePwd);
			    	ssf = new SSLSocketFactory(scf.getSslContext(true));
		    	}
		    	else {
		    		ssf = new SSLSocketFactory(SSLContext.getDefault());
		    	}
		    	Scheme secureScheme = new Scheme("https", port, ssf);
		    	schemeReg = new SchemeRegistry();
		    	schemeReg.register(secureScheme);
		    }
			route = new HttpRoute(httpHost, null, httpProxy, secure, RouteInfo.TunnelType.PLAIN, RouteInfo.LayerType.PLAIN);
			if (schemeReg != null) {
				connMgr = new BasicClientConnectionManager(schemeReg);
			} else {
				connMgr = new BasicClientConnectionManager();
			}
			connReq = connMgr.requestConnection(route, null);
			connection = connReq.getConnection(0, null);
			connection.open(route, null, new BasicHttpParams());
			logger.log(OpLevel.DEBUG, "Connected to {0}{1}", uri, (httpProxy != null ? " via proxy " + httpProxy : ""));
		} catch (Exception e) {
			close();
			throw new IOException("Failed to connect to uri=" + uri + ", reason=" + e.getMessage(), e);
		}
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void connect(String token) throws IOException {
		connect();
		if (!StringUtils.isEmpty(token)) {
			AuthUtils.authenticate(this, token);
			logger.log(OpLevel.DEBUG, "Authenticated connection={0} with token={1}", this, Utils.hide(token, "x", 4));
		}
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void sendRequest(HttpRequest request, boolean wantResponse) throws IOException {
		if (!(request instanceof org.apache.http.HttpRequest))
			throw new IllegalArgumentException("request must be an instance of org.apache.http.HttpRequest");

		try {
			if (!wantResponse)
				request.setHeader(NO_RESPONSE_REQUIRED_HEADER, NO_RESPONSE_REQUIRED_VALUE);
<commit_msg>avoid NPE with newer HTTP client libs<commit_after>/*
 * Copyright 2015 JKOOL, LLC.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.jkool.jesl.net.http;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;

import javax.net.ssl.SSLContext;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpException;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.ProtocolVersion;
import org.apache.http.conn.ClientConnectionRequest;
import org.apache.http.conn.ManagedClientConnection;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.routing.RouteInfo;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.conn.BasicClientConnectionManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.protocol.BasicHttpContext;

import com.jkool.jesl.net.http.apache.HttpRequestImpl;
import com.jkool.jesl.net.http.apache.HttpResponseImpl;
import com.jkool.jesl.net.security.AccessResponse;
import com.jkool.jesl.net.security.AuthUtils;
import com.jkool.jesl.net.ssl.SSLContextFactory;
import com.nastel.jkool.tnt4j.core.OpLevel;
import com.nastel.jkool.tnt4j.sink.DefaultEventSinkFactory;
import com.nastel.jkool.tnt4j.sink.EventSink;
import com.nastel.jkool.tnt4j.utils.Utils;

/**
 * This class provides HTTP[S] connection to the specified JESK server based on given URL.
 *
 * @version $Revision: 3 $
 */
public class HttpClient implements HttpStream {
	public static final String NO_RESPONSE_REQUIRED_HEADER = "Pragma";
	public static final String NO_RESPONSE_REQUIRED_VALUE  = "no-response";

	protected URI							uri;
	protected EventSink		 			    logger;
	protected String						host;
	protected int							port;
	protected String						uriPath;
	protected String						sslKeystore;
	protected String						sslKeystorePwd;
	protected HttpHost						httpHost;
	protected HttpHost						httpProxy;
	protected BasicClientConnectionManager	connMgr;
	protected ClientConnectionRequest		connReq;
	protected ManagedClientConnection		connection;
	protected HttpRoute						route;
	protected SchemeRegistry				schemeReg;
	protected boolean						secure = false;

	/**
	 * Create JESL HTTP[S} client stream with given attributes
	 *
	 * @param urlStr connection string to specified JESL server
	 * @param logger event sink used for logging, null if none
	 * @throws URISyntaxException if invalid connection string
	 */
	public HttpClient(String urlStr, EventSink logger) throws URISyntaxException {
		this(urlStr, null, 0, logger);
	}

	/**
	 * Create JESL HTTP[S} client stream with given attributes
	 *
	 * @param urlStr connection string to specified JESL server
	 * @param proxyHost proxy host name if any, null if none
	 * @param proxyPort proxy port number if any, 0 of none
	 * @param logger event sink used for logging, null if none
	 * @throws URISyntaxException if invalid connection string
	 */
	public HttpClient(String urlStr, String proxyHost, int proxyPort, EventSink logger) throws URISyntaxException {
		uri = new URI(urlStr);
		String scheme = uri.getScheme();
		secure = "https".equalsIgnoreCase(scheme);
		host = uri.getHost();
	    if (host == null)
	    	host = "localhost";
	    port = uri.getPort();
	    if (port <= 0)
	    	port = (secure ? 443 : 80);
		init(host, port, uri.getPath(), secure, proxyHost, proxyPort, logger);
	}

	/**
	 *
	 */
	private void init(String host, int port, String uriPath, boolean secure,
					  String proxyHost, int proxyPort, EventSink logger) {
		this.host    = host;
		this.port    = port;
		this.uriPath = uriPath;
		this.secure  = secure;
		this.logger  = (logger != null ? logger : DefaultEventSinkFactory.defaultEventSink(HttpClient.class));

		String scheme = (secure ? "https" : "http");

	    httpHost = new HttpHost(host, port, scheme);
		if (!StringUtils.isEmpty(proxyHost))
			httpProxy = new HttpHost(proxyHost, proxyPort);
	}

	/**
	 * Sets keystore and password for obtaining SSL certificate for server.
	 *
	 * @param sslKeystore path to keystore
	 * @param sslKeystorePwd password for keystore
	 */
	public void setSslKeystore(String sslKeystore, String sslKeystorePwd) {
		this.sslKeystore    = sslKeystore;
		this.sslKeystorePwd = sslKeystorePwd;
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void connect() throws IOException {
		try {
		    if (secure) {
		    	SSLSocketFactory ssf = null;
		    	if (!StringUtils.isEmpty(sslKeystore)) {
			    	SSLContextFactory scf = new SSLContextFactory(sslKeystore, sslKeystorePwd, sslKeystorePwd);
			    	ssf = new SSLSocketFactory(scf.getSslContext(true));
		    	}
		    	else {
		    		ssf = new SSLSocketFactory(SSLContext.getDefault());
		    	}
		    	Scheme secureScheme = new Scheme("https", port, ssf);
		    	schemeReg = new SchemeRegistry();
		    	schemeReg.register(secureScheme);
		    }
			route = new HttpRoute(httpHost, null, httpProxy, secure, RouteInfo.TunnelType.PLAIN, RouteInfo.LayerType.PLAIN);
			if (schemeReg != null) {
				connMgr = new BasicClientConnectionManager(schemeReg);
			} else {
				connMgr = new BasicClientConnectionManager();
			}
			connReq = connMgr.requestConnection(route, null);
			connection = connReq.getConnection(0, null);
			connection.open(route, new BasicHttpContext(), new BasicHttpParams());
			logger.log(OpLevel.DEBUG, "Connected to {0}{1}", uri, (httpProxy != null ? " via proxy " + httpProxy : ""));
		} catch (Throwable e) {
			close();
			throw new IOException("Failed to connect to uri=" + uri + ", reason=" + e.getMessage(), e);
		}
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void connect(String token) throws IOException {
		connect();
		if (!StringUtils.isEmpty(token)) {
			AuthUtils.authenticate(this, token);
			logger.log(OpLevel.DEBUG, "Authenticated connection={0} with token={1}", this, Utils.hide(token, "x", 4));
		}
	}

	/**
	 * {@inheritDoc}
	 */
	@Override
	public synchronized void sendRequest(HttpRequest request, boolean wantResponse) throws IOException {
		if (!(request instanceof org.apache.http.HttpRequest))
			throw new IllegalArgumentException("request must be an instance of org.apache.http.HttpRequest");

		try {
			if (!wantResponse)
				request.setHeader(NO_RESPONSE_REQUIRED_HEADER, NO_RESPONSE_REQUIRED_VALUE);

{'fraction_non_alphanumeric': 0.10130672027570362, 'fraction_numerical': 0.00402067777139575, 'mean_word_length': 4.098462664714495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '242372', 'n_tokens_mistral': 4610, 'n_tokens_neox': 4188, 'n_words': 1167}