<commit_before>package com.commafeed.backend;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.util.EntityUtils;

public class HttpGetter {

	public String get(String url) throws Exception {
		return new String(getBinary(url), "UTF-8");
	}

	public byte[] getBinary(String url) throws Exception {
		byte[] content = null;

		HttpClient httpclient = new DefaultHttpClient();
		HttpProtocolParams.setContentCharset(httpclient.getParams(), "UTF-8");
		HttpConnectionParams
				.setConnectionTimeout(httpclient.getParams(), 15000);
		HttpConnectionParams.setSoTimeout(httpclient.getParams(), 15000);

		try {
			HttpGet httpget = new HttpGet(url);
			HttpResponse response = httpclient.execute(httpget);
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				content = EntityUtils.toByteArray(entity);
			}
		} finally {
			httpclient.getConnectionManager().shutdown();
		}
		return content;<commit_msg>ignore cookies<commit_after>package com.commafeed.backend;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.params.HttpClientParams;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.util.EntityUtils;

public class HttpGetter {

	public String get(String url) throws Exception {
		return new String(getBinary(url), "UTF-8");
	}

	public byte[] getBinary(String url) throws Exception {
		byte[] content = null;

		HttpClient httpclient = new DefaultHttpClient();
		HttpParams params = httpclient.getParams();
		HttpClientParams.setCookiePolicy(params, CookiePolicy.IGNORE_COOKIES);
		HttpProtocolParams.setContentCharset(params, "UTF-8");
		HttpConnectionParams.setConnectionTimeout(params, 5000);
		HttpConnectionParams.setSoTimeout(params, 5000);

		try {
			HttpGet httpget = new HttpGet(url);
			HttpResponse response = httpclient.execute(httpget);
			HttpEntity entity = response.getEntity();
			if (entity != null) {
				content = EntityUtils.toByteArray(entity);
			}
		} finally {
			httpclient.getConnectionManager().shutdown();
		}
		return content;
{'fraction_non_alphanumeric': 0.11127990884922143, 'fraction_numerical': 0.008355488036460312, 'mean_word_length': 5.503703703703704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6981715', 'n_tokens_mistral': 847, 'n_tokens_neox': 766, 'n_words': 152}