// Path: src/main/java/crawlercommons/robots/SimpleRobotRules.java
// public enum RobotRulesMode {
//     ALLOW_ALL, ALLOW_NONE, ALLOW_SOME
// }


import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
    // number of warnings found in the latest processed robots.txt file
    private ThreadLocal<Integer> _numWarningsDuringLastParse = new ThreadLocal<>();

    private int _maxWarnings;
    private long _maxCrawlDelay;

    public SimpleRobotRulesParser() {
        this(DEFAULT_MAX_CRAWL_DELAY, DEFAULT_MAX_WARNINGS);
    }

    /**
     * @param maxCrawlDelay
     *            see {@link #setMaxCrawlDelay(long)}
     * @param maxWarnings
     *            see {@link #setMaxWarnings(int)}
     */
    public SimpleRobotRulesParser(long maxCrawlDelay, int maxWarnings) {
        this._maxCrawlDelay = maxCrawlDelay;
        this._maxWarnings = maxWarnings;
    }

    @Override
    public SimpleRobotRules failedFetch(int httpStatusCode) {
        SimpleRobotRules result;

        if ((httpStatusCode >= 200) && (httpStatusCode < 300)) {
            throw new IllegalStateException("Can't use status code constructor with 2xx response");
        } else if ((httpStatusCode >= 300) && (httpStatusCode < 400)) {
            // Should only happen if we're getting endless redirects (more than
            // our follow limit), so treat it as a temporary failure.
// Path: src/main/java/crawlercommons/robots/SimpleRobotRules.java
// public enum RobotRulesMode {
//     ALLOW_ALL, ALLOW_NONE, ALLOW_SOME
// }


// Path: src/main/java/crawlercommons/robots/SimpleRobotRulesParser.java
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

    // number of warnings found in the latest processed robots.txt file
    private ThreadLocal<Integer> _numWarningsDuringLastParse = new ThreadLocal<>();

    private int _maxWarnings;
    private long _maxCrawlDelay;

    public SimpleRobotRulesParser() {
        this(DEFAULT_MAX_CRAWL_DELAY, DEFAULT_MAX_WARNINGS);
    }

    /**
     * @param maxCrawlDelay
     *            see {@link #setMaxCrawlDelay(long)}
     * @param maxWarnings
     *            see {@link #setMaxWarnings(int)}
     */
    public SimpleRobotRulesParser(long maxCrawlDelay, int maxWarnings) {
        this._maxCrawlDelay = maxCrawlDelay;
        this._maxWarnings = maxWarnings;
    }

    @Override
    public SimpleRobotRules failedFetch(int httpStatusCode) {
        SimpleRobotRules result;

        if ((httpStatusCode >= 200) && (httpStatusCode < 300)) {
            throw new IllegalStateException("Can't use status code constructor with 2xx response");
        } else if ((httpStatusCode >= 300) && (httpStatusCode < 400)) {
            // Should only happen if we're getting endless redirects (more than
            // our follow limit), so treat it as a temporary failure.
            result = new SimpleRobotRules(RobotRulesMode.ALLOW_NONE);