// Path: src/main/java/crawlercommons/utils/Strings.java
// public class Strings {
// 
//     public static boolean isBlank(final String cs) {
//         if (cs == null || cs.isEmpty()) {
//             return true;
//         }
//         for (int i = 0; i < cs.length(); i++) {
//             if (Character.isWhitespace(cs.charAt(i)) == false) {
//                 return false;
//             }
//         }
//         return true;
//     }
// 
// }
// 
// Path: src/main/java/crawlercommons/filters/URLFilter.java
// public abstract class URLFilter {
// 
//     /**
//      * Returns a modified version of the input URL or null if the URL should be
//      * removed
//      * 
//      * @param urlString
//      *            a URL string to check against filter(s)
//      * @return a filtered URL
//      **/
//     public abstract String filter(String urlString);
// 
// }


import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import crawlercommons.filters.URLFilter;
import static java.nio.charset.StandardCharsets.UTF_8;
import crawlercommons.utils.Strings;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;
    /**
     * Parses the URL file and applies normalizations to the path and query components.
     *
     * @param file the URL file (as in java.net.URL.getFile()).
     * @return a normalized URL file
     */
    private String normalizeUrlFile(String file) {
        // find the beginning of the query parameters
        int endPathIdx = file.indexOf('?');
        if (endPathIdx == -1) {
            // no query parameters, just properly normalize the path
            return escapePath(unescapePath(file));
        }

        int queryStartIdx = endPathIdx + 1;
        if (queryStartIdx >= file.length()) {
            // question mark was the last char in the file, so the query parameters
            // string is empty. we can just remove the question mark and properly
            // normalize the path.
            final String path = file.substring(0, file.length() - 1);
            return escapePath(unescapePath(path));
        }

        file = escapePath(unescapePath(file));

        List<NameValuePair> pairs =
                parseQueryParameters(file, queryStartIdx, queryParamsToRemove);

        StringBuilder normalizedFile = new StringBuilder();
        String path = file.substring(0, endPathIdx);
// Path: src/main/java/crawlercommons/utils/Strings.java
// public class Strings {
// 
//     public static boolean isBlank(final String cs) {
//         if (cs == null || cs.isEmpty()) {
//             return true;
//         }
//         for (int i = 0; i < cs.length(); i++) {
//             if (Character.isWhitespace(cs.charAt(i)) == false) {
//                 return false;
//             }
//         }
//         return true;
//     }
// 
// }
// 
// Path: src/main/java/crawlercommons/filters/URLFilter.java
// public abstract class URLFilter {
// 
//     /**
//      * Returns a modified version of the input URL or null if the URL should be
//      * removed
//      * 
//      * @param urlString
//      *            a URL string to check against filter(s)
//      * @return a filtered URL
//      **/
//     public abstract String filter(String urlString);
// 
// }


// Path: src/main/java/crawlercommons/filters/basic/BasicURLNormalizer.java
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import crawlercommons.filters.URLFilter;
import static java.nio.charset.StandardCharsets.UTF_8;
import crawlercommons.utils.Strings;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.IDN;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.*;

    /**
     * Parses the URL file and applies normalizations to the path and query components.
     *
     * @param file the URL file (as in java.net.URL.getFile()).
     * @return a normalized URL file
     */
    private String normalizeUrlFile(String file) {
        // find the beginning of the query parameters
        int endPathIdx = file.indexOf('?');
        if (endPathIdx == -1) {
            // no query parameters, just properly normalize the path
            return escapePath(unescapePath(file));
        }

        int queryStartIdx = endPathIdx + 1;
        if (queryStartIdx >= file.length()) {
            // question mark was the last char in the file, so the query parameters
            // string is empty. we can just remove the question mark and properly
            // normalize the path.
            final String path = file.substring(0, file.length() - 1);
            return escapePath(unescapePath(path));
        }

        file = escapePath(unescapePath(file));

        List<NameValuePair> pairs =
                parseQueryParameters(file, queryStartIdx, queryParamsToRemove);

        StringBuilder normalizedFile = new StringBuilder();
        String path = file.substring(0, endPathIdx);
        if (!Strings.isBlank(path)) {