<commit_before>/******************************************************
 * Web crawler
 * 
 *
 * Copyright (C) 2012 by <NAME> (http://peterhedenskog.com)
 *
 ******************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
 * compliance with the License. You may obtain a copy of the License at
 * 
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is 
 * distributed  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   
 * See the License for the specific language governing permissions and limitations under the License.
 *
 *******************************************************
 */
package com.soulgalore.crawler.core.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;

import com.google.inject.Inject;
import com.soulgalore.crawler.core.PageURL;
import com.soulgalore.crawler.core.HTMLPageResponse;
import com.soulgalore.crawler.core.HTMLPageResponseFetcher;
import com.soulgalore.crawler.util.StatusCode;

/**
 * Fetch urls by a HTTPClient. Note: Will only fetch response headers for
 * resources that fails and for pages (meaning where the body of the response is
 * fetched).
 * 
 * 
 */
public class HttpClientResponseFetcher implements HTMLPageResponseFetcher {

	private final HttpClient httpClient;

	/**
	 * Create a new fetcher.
	 * 
	 * @param client
	 *            the client to use
	 */
	@Inject
	public HttpClientResponseFetcher(HttpClient client) {
		httpClient = client;
	}

	/**
	 * Shutdown the client.
	 */
	public void shutdown() {
		httpClient.getConnectionManager().shutdown();
	}

	/**
	 * Get a response.
	 * @param url the url
	 * @param getPage the body of the page or not
	 * @return the response
	 */
	public HTMLPageResponse get(PageURL url, boolean getPage) {

		if (url.isWrongSyntax()) {
			return new HTMLPageResponse(url, StatusCode.SC_MALFORMED_URI,
					Collections.<String, String>emptyMap(), "", "", 0);
		}
	
		final HttpGet get = new HttpGet(url.getUri());

		try {

			final HttpResponse resp = httpClient.execute(get);
			final HttpEntity entity = resp.getEntity();

			
			if (entity.getContentType() == null
					|| (!entity.getContentType().toString().contains("html"))) {
				return new HTMLPageResponse(url,
						StatusCode.SC_WRONG_CONTENT_TYPE,
						Collections.<String, String>emptyMap(), "", "", 0);
			}
			

			// this is a hack to minimize the amount of memory used
			// should make this configurable maybe
			// don't fetch headers for request that don't fetch the body and
			// response isn't 200
			// these headers will not be shown in the results
			final Map<String, String> headersAndValues = getPage
					|| !StatusCode.getInstance().isResponseCodeOk(
							resp.getStatusLine().getStatusCode()) ? getHeaders(resp)
					: Collections.<String, String>emptyMap();

			final String encoding = entity.getContentEncoding() != null ? entity
					.getContentEncoding().getValue() : "";

			final String body = getPage ? getBody(entity, "".equals(encoding)?"UTF-8":encoding) : "";
			final long size = entity.getContentLength();

			EntityUtils.consume(entity);
			return new HTMLPageResponse(url, resp.getStatusLine()
					.getStatusCode(), headersAndValues, body, encoding, size);

		} catch (SocketTimeoutException e) {
			return new HTMLPageResponse(url,
					StatusCode.SC_SERVER_RESPONSE_TIMEOUT,
					Collections.<String, String>emptyMap(), "", "", 0);
			}

		catch (IOException e) {
			System.out.println(e);
			return new HTMLPageResponse(url,
					StatusCode.SC_SERVER_RESPONSE_UNKNOWN,
					Collections.<String, String>emptyMap(), "", "", 0);
		}

	}

	/**
	 * Get the body.
	 * 
	 * @param entity
	 *            the http entity from the response
	 * @param enc
	 *            the encoding
	 * @return the body as a String
	 * @throws IOException
	 *             if the body couldn't be fetched<commit_msg>fixed connection closing issue<commit_after>/******************************************************
 * Web crawler
 * 
 *
 * Copyright (C) 2012 by <NAME> (http://peterhedenskog.com)
 *
 ******************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
 * compliance with the License. You may obtain a copy of the License at
 * 
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is 
 * distributed  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   
 * See the License for the specific language governing permissions and limitations under the License.
 *
 *******************************************************
 */
package com.soulgalore.crawler.core.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;

import com.google.inject.Inject;
import com.soulgalore.crawler.core.PageURL;
import com.soulgalore.crawler.core.HTMLPageResponse;
import com.soulgalore.crawler.core.HTMLPageResponseFetcher;
import com.soulgalore.crawler.util.StatusCode;

/**
 * Fetch urls by a HTTPClient. Note: Will only fetch response headers for
 * resources that fails and for pages (meaning where the body of the response is
 * fetched).
 * 
 * 
 */
public class HttpClientResponseFetcher implements HTMLPageResponseFetcher {

	private final HttpClient httpClient;

	/**
	 * Create a new fetcher.
	 * 
	 * @param client
	 *            the client to use
	 */
	@Inject
	public HttpClientResponseFetcher(HttpClient client) {
		httpClient = client;
	}

	/**
	 * Shutdown the client.
	 */
	public void shutdown() {
		httpClient.getConnectionManager().shutdown();
	}

	/**
	 * Get a response.
	 * @param url the url
	 * @param getPage the body of the page or not
	 * @return the response
	 */
	public HTMLPageResponse get(PageURL url, boolean getPage) {

		if (url.isWrongSyntax()) {
			return new HTMLPageResponse(url, StatusCode.SC_MALFORMED_URI,
					Collections.<String, String>emptyMap(), "", "", 0);
		}
	
		final HttpGet get = new HttpGet(url.getUri());
		HttpEntity entity = null;
		
		try {

			final HttpResponse resp = httpClient.execute(get);
			entity = resp.getEntity();

			
			if (entity.getContentType() == null
					|| (!entity.getContentType().toString().contains("html"))) {
				return new HTMLPageResponse(url,
						StatusCode.SC_WRONG_CONTENT_TYPE,
						Collections.<String, String>emptyMap(), "", "", 0);
			}
			

			// this is a hack to minimize the amount of memory used
			// should make this configurable maybe
			// don't fetch headers for request that don't fetch the body and
			// response isn't 200
			// these headers will not be shown in the results
			final Map<String, String> headersAndValues = getPage
					|| !StatusCode.getInstance().isResponseCodeOk(
							resp.getStatusLine().getStatusCode()) ? getHeaders(resp)
					: Collections.<String, String>emptyMap();

			final String encoding = entity.getContentEncoding() != null ? entity
					.getContentEncoding().getValue() : "";

			final String body = getPage ? getBody(entity, "".equals(encoding)?"UTF-8":encoding) : "";
			final long size = entity.getContentLength();

			return new HTMLPageResponse(url, resp.getStatusLine()
					.getStatusCode(), headersAndValues, body, encoding, size);

		} catch (SocketTimeoutException e) {
			return new HTMLPageResponse(url,
					StatusCode.SC_SERVER_RESPONSE_TIMEOUT,
					Collections.<String, String>emptyMap(), "", "", 0);
			}

		catch (IOException e) {
			System.out.println(e);
			return new HTMLPageResponse(url,
					StatusCode.SC_SERVER_RESPONSE_UNKNOWN,
					Collections.<String, String>emptyMap(), "", "", 0);
		}
		finally {
			try {
				EntityUtils.consume(entity);
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		

	}

	/**
	 * Get the body.
	 * 
	 * @param entity
	 *            the http entity from the response
	 * @param enc
	 *            the encoding
	 * @return the body as a String
	 * @throws IOException
	 *             if the body couldn't be fetched
{'fraction_non_alphanumeric': 0.1411464685663599, 'fraction_numerical': 0.003548065195697971, 'mean_word_length': 3.732423924449108, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6990249', 'n_tokens_mistral': 2871, 'n_tokens_neox': 2522, 'n_words': 751}