// Path: access-control/src/main/java/org/archive/accesscontrol/RobotsUnavailableException.java
// public class RobotsUnavailableException extends AccessControlException {
// 
//     /**
//      * 
//      */
//     private static final long serialVersionUID = -6268896797166951256L;
// 
//     public RobotsUnavailableException() {
//         super();
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(String arg0, Throwable arg1) {
//         super(arg0, arg1);
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(String arg0) {
//         super(arg0);
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(Throwable arg0) {
//         super(arg0);
//         // TODO Auto-generated constructor stub
//     }
// 
// }


import java.io.IOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.logging.Logger;
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.archive.accesscontrol.RobotsUnavailableException;
package org.archive.accesscontrol.robotstxt;



/**
 * HttpRobotClient allows fetching of robots.txt rules over HTTP.
 * 
 * @author aosborne
 *
 */
public class HttpRobotClient extends RobotClient {
    private static final Logger LOGGER = Logger.getLogger(
            RobotClient.class.getName());
    protected HttpClient http = new HttpClient(
            new MultiThreadedHttpConnectionManager());
    
    public HttpClient getHttpClient() {
        return http;
    }

// Path: access-control/src/main/java/org/archive/accesscontrol/RobotsUnavailableException.java
// public class RobotsUnavailableException extends AccessControlException {
// 
//     /**
//      * 
//      */
//     private static final long serialVersionUID = -6268896797166951256L;
// 
//     public RobotsUnavailableException() {
//         super();
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(String arg0, Throwable arg1) {
//         super(arg0, arg1);
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(String arg0) {
//         super(arg0);
//         // TODO Auto-generated constructor stub
//     }
// 
//     public RobotsUnavailableException(Throwable arg0) {
//         super(arg0);
//         // TODO Auto-generated constructor stub
//     }
// 
// }


// Path: access-control/src/main/java/org/archive/accesscontrol/robotstxt/HttpRobotClient.java
import java.io.IOException;
import java.net.ConnectException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.logging.Logger;
import org.apache.commons.httpclient.ConnectTimeoutException;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.GetMethod;
import org.archive.accesscontrol.RobotsUnavailableException;

package org.archive.accesscontrol.robotstxt;



/**
 * HttpRobotClient allows fetching of robots.txt rules over HTTP.
 * 
 * @author aosborne
 *
 */
public class HttpRobotClient extends RobotClient {
    private static final Logger LOGGER = Logger.getLogger(
            RobotClient.class.getName());
    protected HttpClient http = new HttpClient(
            new MultiThreadedHttpConnectionManager());
    
    public HttpClient getHttpClient() {
        return http;
    }

    public RobotRules getRulesForUrl(String url, String userAgent) throws IOException, RobotsUnavailableException {