<commit_before>/**
 *
 */
package ecologylab.bigsemantics.downloaders.controllers;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.LinkedHashMap;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;

import ecologylab.bigsemantics.collecting.SemanticsGlobalScope;
import ecologylab.bigsemantics.documentparsers.DocumentParser;
import ecologylab.bigsemantics.downloaders.LocalDocumentCache;
import ecologylab.bigsemantics.downloaders.oodss.DownloadResponse;
import ecologylab.bigsemantics.filestorage.FileMetadata;
import ecologylab.bigsemantics.filestorage.FileStorageProvider;
import ecologylab.bigsemantics.filestorage.FileSystemStorage;
import ecologylab.bigsemantics.metadata.builtins.Document;
import ecologylab.bigsemantics.metadata.builtins.DocumentClosure;
import ecologylab.concurrent.DownloadableLogRecord;
import ecologylab.generic.Debug;
import ecologylab.net.PURLConnection;
import ecologylab.net.ParsedURL;
import ecologylab.serialization.SimplTypesScope;
import ecologylab.serialization.formatenums.StringFormat;

/**
 * HTTP download controller request network download to an HTTP server
 * 
 * @author ajit
 * 
 */

public class HTTPDownloadController extends Debug implements DownloadController
{

	public static int									HTTP_DOWNLOAD_REQUEST_TIMEOUT	= 45000;

	private static final String				SERVICE_LOC										= "http://localhost:8080/BigSemanticsService/download?";

	private static SimplTypesScope		tscope;

	LinkedHashMap<ParsedURL, Boolean>	recentlyCached;

	Object														lockRecentlyCached						= new Object();

	// private static Logger htmlCacheLog = Logger.getLogger(BaseLogger.htmlCacheLogger);

	static
	{
		tscope = SimplTypesScope.get("deserialize-download-response", DownloadResponse.class);
	}

	public HTTPDownloadController()
	{
		recentlyCached = new LinkedHashMap<ParsedURL, Boolean>(1000);
	}

	@Override
	public void connect(DocumentClosure documentClosure) throws IOException
	{
		Document document = documentClosure.getDocument();<commit_msg>remove port in localhost request uri<commit_after>/**
 *
 */
package ecologylab.bigsemantics.downloaders.controllers;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.LinkedHashMap;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;

import ecologylab.bigsemantics.collecting.SemanticsGlobalScope;
import ecologylab.bigsemantics.documentparsers.DocumentParser;
import ecologylab.bigsemantics.downloaders.LocalDocumentCache;
import ecologylab.bigsemantics.downloaders.oodss.DownloadResponse;
import ecologylab.bigsemantics.filestorage.FileMetadata;
import ecologylab.bigsemantics.filestorage.FileStorageProvider;
import ecologylab.bigsemantics.filestorage.FileSystemStorage;
import ecologylab.bigsemantics.metadata.builtins.Document;
import ecologylab.bigsemantics.metadata.builtins.DocumentClosure;
import ecologylab.concurrent.DownloadableLogRecord;
import ecologylab.generic.Debug;
import ecologylab.net.PURLConnection;
import ecologylab.net.ParsedURL;
import ecologylab.serialization.SimplTypesScope;
import ecologylab.serialization.formatenums.StringFormat;

/**
 * HTTP download controller request network download to an HTTP server
 * 
 * @author ajit
 * 
 */

public class HTTPDownloadController extends Debug implements DownloadController
{

	public static int									HTTP_DOWNLOAD_REQUEST_TIMEOUT	= 45000;

	private static final String				SERVICE_LOC										= "http://localhost/BigSemanticsService/download?";

	private static SimplTypesScope		tscope;

	LinkedHashMap<ParsedURL, Boolean>	recentlyCached;

	Object														lockRecentlyCached						= new Object();

	// private static Logger htmlCacheLog = Logger.getLogger(BaseLogger.htmlCacheLogger);

	static
	{
		tscope = SimplTypesScope.get("deserialize-download-response", DownloadResponse.class);
	}

	public HTTPDownloadController()
	{
		recentlyCached = new LinkedHashMap<ParsedURL, Boolean>(1000);
	}

	@Override
	public void connect(DocumentClosure documentClosure) throws IOException
	{
		Document document = documentClosure.getDocument();
{'fraction_non_alphanumeric': 0.08134738771769019, 'fraction_numerical': 0.005041246562786435, 'mean_word_length': 6.040322580645161, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5532830', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1301, 'n_words': 234}