<commit_before>/*
 * This file is part of Flicklib.
 *
 * Copyright (C) <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.flicklib.service;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.flicklib.tools.IOTools;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;

/**
 * Loads a http request
 * 
 * @author francisdb
 */
@Singleton
public class HttpSourceLoader implements SourceLoader {

    private static final Logger LOGGER = LoggerFactory.getLogger(HttpSourceLoader.class);

    private HttpClient client;

    @Inject
    public HttpSourceLoader(@Named(value = "http.timeout") final Integer timeout) {
    	// http://hc.apache.org/httpclient-3.x/performance.html#Concurrent_execution_of_HTTP_methods
        client = new HttpClient(new MultiThreadedHttpConnectionManager());
        if (timeout != null) {
            // wait max x sec
            client.getParams().setSoTimeout(timeout);
            //manager.getParams().setSoTimeout(timeout);
            // LOGGER.info("Timeout = "+client.getParams().getSoTimeout());
        }
    }
    
    @Override
    public Source loadSource(String url) throws IOException {
        GetMethod httpMethod = null;
        Reader is = null;
        try {
            LOGGER.info("Loading " + url);
            httpMethod = new GetMethod(url);     
            client.executeMethod(httpMethod);
            LOGGER.info("Finished loading at " + httpMethod.getURI().toString());
            String responseCharset = httpMethod.getResponseCharSet();
            if (responseCharset != null) {
                is = new InputStreamReader(httpMethod.getResponseBodyAsStream(), responseCharset);
            } else {
                is = new InputStreamReader(httpMethod.getResponseBodyAsStream());
            }
            String contentType = httpMethod.getResponseHeader("Content-Type").getValue();
            // String contentType = URLConnection.guessContentTypeFromName(url)
            return new Source(httpMethod.getURI().toString(), IOTools.readerToString(is), contentType);<commit_msg>Catching redirect exceptions like infinite redirect loops<commit_after>/*
 * This file is part of Flicklib.
 *
 * Copyright (C) <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.flicklib.service;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.RedirectException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.flicklib.tools.IOTools;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.name.Named;

/**
 * Loads a http request
 * 
 * @author francisdb
 */
@Singleton
public class HttpSourceLoader implements SourceLoader {

    private static final Logger LOGGER = LoggerFactory.getLogger(HttpSourceLoader.class);

    private HttpClient client;

    @Inject
    public HttpSourceLoader(@Named(value = "http.timeout") final Integer timeout) {
    	// http://hc.apache.org/httpclient-3.x/performance.html#Concurrent_execution_of_HTTP_methods
        client = new HttpClient(new MultiThreadedHttpConnectionManager());
        if (timeout != null) {
            // wait max x sec
            client.getParams().setSoTimeout(timeout);
            //manager.getParams().setSoTimeout(timeout);
            // LOGGER.info("Timeout = "+client.getParams().getSoTimeout());
        }
    }
    
    @Override
    public Source loadSource(String url) throws IOException {
        GetMethod httpMethod = null;
        Reader is = null;
        try {
            LOGGER.info("Loading " + url);
            httpMethod = new GetMethod(url);   
            try{
            	client.executeMethod(httpMethod);
            }catch(RedirectException ex){
            	throw new IOException("Redirect problem: "+ex.getMessage(), ex);
            }
            LOGGER.info("Finished loading at " + httpMethod.getURI().toString());
            String responseCharset = httpMethod.getResponseCharSet();
            if (responseCharset != null) {
                is = new InputStreamReader(httpMethod.getResponseBodyAsStream(), responseCharset);
            } else {
                is = new InputStreamReader(httpMethod.getResponseBodyAsStream());
            }
            String contentType = httpMethod.getResponseHeader("Content-Type").getValue();
            // String contentType = URLConnection.guessContentTypeFromName(url)
            return new Source(httpMethod.getURI().toString(), IOTools.readerToString(is), contentType);
{'fraction_non_alphanumeric': 0.09831738700098977, 'fraction_numerical': 0.0023094688221709007, 'mean_word_length': 3.175619834710744, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2492894', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1582, 'n_words': 430}