<commit_before>/**
 * Copyright (C) 2012-2014 Gist Labs, LLC. (http://gistlabs.com)
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */
package com.gistlabs.mechanize.impl;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;

import com.gistlabs.mechanize.Mechanize;
import com.gistlabs.mechanize.Resource;
import com.gistlabs.mechanize.ResourceFactory;
import com.gistlabs.mechanize.cache.HttpCacheFilter;
import com.gistlabs.mechanize.cache.api.HttpCache;
import com.gistlabs.mechanize.cache.inMemory.InMemoryHttpCache;
import com.gistlabs.mechanize.cookie.Cookies;
import com.gistlabs.mechanize.exceptions.MechanizeExceptionFactory;
import com.gistlabs.mechanize.filters.DefaultMechanizeChainFilter;
import com.gistlabs.mechanize.filters.MechanizeChainFilter;
import com.gistlabs.mechanize.parameters.Parameters;
import com.gistlabs.mechanize.requestor.PageRequestor;
import com.gistlabs.mechanize.requestor.RequestBuilder;
import com.gistlabs.mechanize.requestor.RequestBuilderFactory;
import com.gistlabs.mechanize.util.apache.ContentType;

/**
 * Mechanize agent acts as a focal point for HTTP interactions and also as a factory for Page objects from responses.
 *
 * <p>Interesting resources: http://en.wikipedia.org/wiki/List_of_HTTP_header_fields</p>
 *
 * <p>NOTE: The mechanize library is not synchronized and should be used in a single thread environment or with custom synchronization.</p>
 *
 * @author <NAME><<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */
public class MechanizeAgent implements PageRequestor<Resource>, RequestBuilderFactory<Resource>, Mechanize {

	static final Map<String,ResourceFactory> PAGE_FACTORIES = new HashMap<String, ResourceFactory>();

	static ResourceFactory lookupFactory(final String mimeType) {
		return PAGE_FACTORIES.get(mimeType);
	}
	static void registerFactory(final ResourceFactory factory) {
		Collection<String> contentMatches = factory.getContentMatches();
		for (String mimeType : contentMatches)
			PAGE_FACTORIES.put(mimeType, factory);
	}

	static String VERSION;
	public static void setVersion(final String version) {
		VERSION=version;
	}

	static {
		MechanizeInitializer.initialize();
	}

	private final DefaultMechanizeChainFilter requestChain;
	private final AbstractHttpClient client;
	private final Cookies cookies;

	public MechanizeAgent() {
		this(buildDefaultHttpClient());
	}

	public MechanizeAgent(final AbstractHttpClient client) {
		this(client, new InMemoryHttpCache());
	}

	public MechanizeAgent(final HttpCache httpCache) {
		this(buildDefaultHttpClient(), httpCache);
	}

	public MechanizeAgent(final AbstractHttpClient client, final HttpCache httpCache) {
		this.client = client;
		setupClient(client);

		this.requestChain = new DefaultMechanizeChainFilter(new MechanizeHttpClientFilter(this.client));
		addFilter(new HttpCacheFilter(httpCache));

		this.cookies = new Cookies(this.client);

	}

	/**
	 * This method is used to capture Location headers after HttpClient redirect handling.
	 */
	private void setupClient(final AbstractHttpClient client) {
		this.client.addResponseInterceptor(new HttpResponseInterceptor() {
			@Override
			public void process(final HttpResponse response, final HttpContext context)
					throws HttpException, IOException {
				Header header = response.getFirstHeader("Location");
				if (header!=null)
					context.setAttribute("Location", header.getValue());
			}
		});<commit_msg>Prepend base URL to URI to prevent MalformedURLExceptions when trying to log in via SFU<commit_after>/**
 * Copyright (C) 2012-2014 Gist Labs, LLC. (http://gistlabs.com)
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */
package com.gistlabs.mechanize.impl;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

import org.apache.http.Header;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.AbstractHttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HttpContext;

import com.gistlabs.mechanize.Mechanize;
import com.gistlabs.mechanize.Resource;
import com.gistlabs.mechanize.ResourceFactory;
import com.gistlabs.mechanize.cache.HttpCacheFilter;
import com.gistlabs.mechanize.cache.api.HttpCache;
import com.gistlabs.mechanize.cache.inMemory.InMemoryHttpCache;
import com.gistlabs.mechanize.cookie.Cookies;
import com.gistlabs.mechanize.exceptions.MechanizeExceptionFactory;
import com.gistlabs.mechanize.filters.DefaultMechanizeChainFilter;
import com.gistlabs.mechanize.filters.MechanizeChainFilter;
import com.gistlabs.mechanize.parameters.Parameters;
import com.gistlabs.mechanize.requestor.PageRequestor;
import com.gistlabs.mechanize.requestor.RequestBuilder;
import com.gistlabs.mechanize.requestor.RequestBuilderFactory;
import com.gistlabs.mechanize.util.apache.ContentType;

/**
 * Mechanize agent acts as a focal point for HTTP interactions and also as a factory for Page objects from responses.
 *
 * <p>Interesting resources: http://en.wikipedia.org/wiki/List_of_HTTP_header_fields</p>
 *
 * <p>NOTE: The mechanize library is not synchronized and should be used in a single thread environment or with custom synchronization.</p>
 *
 * @author <NAME><<EMAIL>>
 * @author <NAME> <<EMAIL>>
 */
public class MechanizeAgent implements PageRequestor<Resource>, RequestBuilderFactory<Resource>, Mechanize {

	static final Map<String,ResourceFactory> PAGE_FACTORIES = new HashMap<String, ResourceFactory>();

	static ResourceFactory lookupFactory(final String mimeType) {
		return PAGE_FACTORIES.get(mimeType);
	}
	static void registerFactory(final ResourceFactory factory) {
		Collection<String> contentMatches = factory.getContentMatches();
		for (String mimeType : contentMatches)
			PAGE_FACTORIES.put(mimeType, factory);
	}

	static String VERSION;
	public static void setVersion(final String version) {
		VERSION=version;
	}

	static {
		MechanizeInitializer.initialize();
	}

	private final DefaultMechanizeChainFilter requestChain;
	private final AbstractHttpClient client;
	private final Cookies cookies;

	public MechanizeAgent() {
		this(buildDefaultHttpClient());
	}

	public MechanizeAgent(final AbstractHttpClient client) {
		this(client, new InMemoryHttpCache());
	}

	public MechanizeAgent(final HttpCache httpCache) {
		this(buildDefaultHttpClient(), httpCache);
	}

	public MechanizeAgent(final AbstractHttpClient client, final HttpCache httpCache) {
		this.client = client;
		setupClient(client);

		this.requestChain = new DefaultMechanizeChainFilter(new MechanizeHttpClientFilter(this.client));
		addFilter(new HttpCacheFilter(httpCache));

		this.cookies = new Cookies(this.client);

	}

	/**
	 * This method is used to capture Location headers after HttpClient redirect handling.
	 */
	private void setupClient(final AbstractHttpClient client) {
		this.client.addResponseInterceptor(new HttpResponseInterceptor() {
			@Override
			public void process(final HttpResponse response, final HttpContext context)
					throws HttpException, IOException {
				Header header = response.getFirstHeader("Location");
				if (header!=null) {
					String location = header.getValue();

					/*
					 * Append the base name to the Location header
					 */
					if (location.startsWith("/")) {
						String baseUrl = context.getAttribute(ExecutionContext.HTTP_TARGET_HOST).toString();
						location = baseUrl + location;
					}
					context.setAttribute("Location", location);
				}
			}
		});
{'fraction_non_alphanumeric': 0.0959409594095941, 'fraction_numerical': 0.0026836632002683663, 'mean_word_length': 5.552380952380952, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '536896', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2612, 'n_words': 619}