<commit_before>/**
 * JLibs: Common Utilities for Java
 * Copyright (C) 2009  <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 */

package jlibs.core.net;

import jlibs.core.io.FileUtil;
import jlibs.core.io.IOUtil;
import jlibs.core.lang.ArrayUtil;
import jlibs.core.lang.ImpossibleException;
import jlibs.core.lang.StringUtil;

import javax.net.ssl.*;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.*;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

/**
 * @author <NAME>
 */
public class URLUtil{
    /**
     * Constructs <code>URI</code> from given string.
     *
     * The <code>URISyntaxException</code> is rethrown as <code>IllegalArgumentException</code>
     */
    public static URI toURI(String str){
        try{
            return new URI(str);
        }catch(URISyntaxException ex){
            throw new IllegalArgumentException(ex);
        }
    }

    /**
     * Constructs <code>URI</code> from given <code>URL</code>.
     *
     * The <code>URISyntaxException</code> is rethrown as <code>IllegalArgumentException</code>
     */
    public static URI toURI(URL url){
        try{
            return url.toURI();
        }catch(URISyntaxException ex){
            throw new IllegalArgumentException(ex);
        }
    }

    public static URL toURL(String systemID){
        if(StringUtil.isWhitespace(systemID))
            return null;
        systemID = systemID.trim();
        try{
            return new URL(systemID);
        }catch(MalformedURLException ex){
            return FileUtil.toURL(new File(systemID));
        }
    }

    public static String toSystemID(URL url){
        try{
            if("file".equals(url.getProtocol()))
                return new File(url.toURI()).getAbsolutePath();
            else
                return url.toString();
        }catch(URISyntaxException ex){
            throw new ImpossibleException(ex);
        }
    }

    public static URI resolve(URL base, URL url){
        return toURI(base).resolve(toURI(url));
    }

    /**
     * returns Query Parameters in specified uri as <code>Map</code>.
     * key will be param name and value wil be param value.
     *
     * @param uri       The string to be parsed into a URI
     * @param encoding  if null, <code>UTF-8</code> will be used
     *
     * @throws URISyntaxException               in case of invalid uri
     * @throws UnsupportedEncodingException     if named character encoding is not supported
     */
    public static Map<String, String> getQueryParams(String uri, String encoding) throws URISyntaxException, UnsupportedEncodingException{
        if(encoding==null)
            encoding = IOUtil.UTF_8.name();

        String query = new URI(uri).getRawQuery();
        String params[] = Pattern.compile("&", Pattern.LITERAL).split(query);
        Map<String, String> map = new HashMap<String, String>(params.length);
        for(String param: params){
            int equal = param.indexOf('=');
            String name = param.substring(0, equal);
            String value = param.substring(equal+1);
            name = URLDecoder.decode(name, encoding);
            value = URLDecoder.decode(value, encoding);
            map.put(name, value);
        }
        return map;
    }

    public static String suggestFile(URI uri, String... extensions){
        if(extensions==null || extensions.length==0)
            throw new IllegalArgumentException("atleast one extension must be specified");

        String path = uri.getPath();
        String tokens[] = StringUtil.getTokens(path, "/", true);
        String file = tokens[tokens.length-1];
        int dot = file.indexOf(".");
        if(dot==-1){
            String query = uri.getQuery();
            if(query!=null){
                query = query.toLowerCase();
                if(ArrayUtil.contains(extensions, query))
                    return file+'.'+query;
            }
            return file+'.'+extensions[0];
        }else if(ArrayUtil.contains(extensions, file.substring(dot+1)))
            return file;
        else
            return file.substring(0, dot+1)+extensions[0];
    }

    public static String suggestPrefix(Properties suggested, String uri){
        String prefix = suggested.getProperty(uri);
        if(prefix!=null)
            return prefix;<commit_msg>suggested file changed to work with new xmlcrawler impl<commit_after>/**
 * JLibs: Common Utilities for Java
 * Copyright (C) 2009  <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 */

package jlibs.core.net;

import jlibs.core.io.FileUtil;
import jlibs.core.io.IOUtil;
import jlibs.core.lang.ImpossibleException;
import jlibs.core.lang.StringUtil;

import javax.net.ssl.*;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.*;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Pattern;

/**
 * @author <NAME>
 */
public class URLUtil{
    /**
     * Constructs <code>URI</code> from given string.
     *
     * The <code>URISyntaxException</code> is rethrown as <code>IllegalArgumentException</code>
     */
    public static URI toURI(String str){
        try{
            return new URI(str);
        }catch(URISyntaxException ex){
            throw new IllegalArgumentException(ex);
        }
    }

    /**
     * Constructs <code>URI</code> from given <code>URL</code>.
     *
     * The <code>URISyntaxException</code> is rethrown as <code>IllegalArgumentException</code>
     */
    public static URI toURI(URL url){
        try{
            return url.toURI();
        }catch(URISyntaxException ex){
            throw new IllegalArgumentException(ex);
        }
    }

    public static URL toURL(String systemID){
        if(StringUtil.isWhitespace(systemID))
            return null;
        systemID = systemID.trim();
        try{
            return new URL(systemID);
        }catch(MalformedURLException ex){
            return FileUtil.toURL(new File(systemID));
        }
    }

    public static String toSystemID(URL url){
        try{
            if("file".equals(url.getProtocol()))
                return new File(url.toURI()).getAbsolutePath();
            else
                return url.toString();
        }catch(URISyntaxException ex){
            throw new ImpossibleException(ex);
        }
    }

    /**
     * returns Query Parameters in specified uri as <code>Map</code>.
     * key will be param name and value wil be param value.
     *
     * @param uri       The string to be parsed into a URI
     * @param encoding  if null, <code>UTF-8</code> will be used
     *
     * @throws URISyntaxException               in case of invalid uri
     * @throws UnsupportedEncodingException     if named character encoding is not supported
     */
    public static Map<String, String> getQueryParams(String uri, String encoding) throws URISyntaxException, UnsupportedEncodingException{
        if(encoding==null)
            encoding = IOUtil.UTF_8.name();

        String query = new URI(uri).getRawQuery();
        String params[] = Pattern.compile("&", Pattern.LITERAL).split(query);
        Map<String, String> map = new HashMap<String, String>(params.length);
        for(String param: params){
            int equal = param.indexOf('=');
            String name = param.substring(0, equal);
            String value = param.substring(equal+1);
            name = URLDecoder.decode(name, encoding);
            value = URLDecoder.decode(value, encoding);
            map.put(name, value);
        }
        return map;
    }

    public static String suggestFile(URI uri, String extension){
        String path = uri.getPath();
        String tokens[] = StringUtil.getTokens(path, "/", true);
        String file = tokens[tokens.length-1];
        int dot = file.indexOf(".");
        if(dot==-1)
            return file+'.'+extension;
        else
            return file.substring(0, dot+1)+extension;
    }

    public static String suggestPrefix(Properties suggested, String uri){
        String prefix = suggested.getProperty(uri);
        if(prefix!=null)
            return prefix;
{'fraction_non_alphanumeric': 0.10512386195214905, 'fraction_numerical': 0.004022866821935211, 'mean_word_length': 2.532909498878085, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4255399', 'n_tokens_mistral': 2587, 'n_tokens_neox': 2542, 'n_words': 765}