<commit_before>module.exports = function (urlCandidate, includeLocalAdressesAndRootUrls) {
    //gracefully taken from https://gist.github.com/dperini/729294 - MIT license
    var isPublicUrlMatcher = new RegExp(
        "^" +
            // protocol identifier
        "(?:(?:https?|ftp)://)" +
            // user:pass authentication
        "(?:\\S+(?::\\S*)?@)?" +
        "(?:" +
            // IP address exclusion
            // private & local networks
        "(?!(?:10|127)(?:\\.\\d{1,3}){3})" +
        "(?!(?:169\\.254|192\\.168)(?:\\.\\d{1,3}){2})" +
        "(?!172\\.(?:1[6-9]|2\\d|3[0-1])(?:\\.\\d{1,3}){2})" +
            // IP address dotted notation octets
            // excludes loopback network 0.0.0.0
            // excludes reserved space >= 172.16.58.3
            // excludes network & broacast addresses
            // (first & last IP address of each class)
        "(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])" +
        "(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}" +
        "(?:\\.(?:[1-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))" +
        "|" +
            // host name
        "(?:(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)" +
            // domain name
        "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)*" +
            // TLD identifier
        "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))" +
            // TLD may end with dot
        "\\.?" +
        ")" +
            // port number
        "(?::\\d{2,5})?" +
            // resource path
        "(?:[/?#]\\S*)?" +
        "$", "i"
    );

    var isLocalAddressOrRootUrlMatcher = new RegExp(
        "^" +
        "(?:" +
            // protocol identifier
        "(?:(?:https?|ftp)://)" +
            // user:pass authentication
        "(?:\\S+(?::\\S*)?@)?" +
            // host name
        "(?:(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)" +
            // domain name
        "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)*" +
            // TLD identifier
        "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))?" +
            // TLD may end with dot
        "\\.?" +
            // port number
        "(?::\\d{2,5})?" +
        ")?" +
            // resource path
        "(?:[/# | \./]\\S*)?" +
        "$", "i"
    );

    var isValidUrl = false;

    if (includeLocalAdressesAndRootUrls) {
        isValidUrl = isLocalAddressOrRootUrlMatcher.test(urlCandidate);
    }

    if (!isValidUrl) {
        isValidUrl = isPublicUrlMatcher.test(urlCandidate);
    }

    return isValidUrl;
};<commit_msg>(fix): improved performance by moving the regex creation to outer scope, so that it will not be created on each function call of isUrl()<commit_after>//gracefully taken from https://gist.github.com/dperini/729294 - MIT license
var isPublicUrlMatcher = new RegExp(
    "^" +
    // protocol identifier
    "(?:(?:https?|ftp)://)" +
    // user:pass authentication
    "(?:\\S+(?::\\S*)?@)?" +
    "(?:" +
    // IP address exclusion
    // private & local networks
    "(?!(?:10|127)(?:\\.\\d{1,3}){3})" +
    "(?!(?:169\\.254|192\\.168)(?:\\.\\d{1,3}){2})" +
    "(?!172\\.(?:1[6-9]|2\\d|3[0-1])(?:\\.\\d{1,3}){2})" +
    // IP address dotted notation octets
    // excludes loopback network 0.0.0.0
    // excludes reserved space >= 172.16.58.3
    // excludes network & broacast addresses
    // (first & last IP address of each class)
    "(?:[1-9]\\d?|1\\d\\d|2[01]\\d|22[0-3])" +
    "(?:\\.(?:1?\\d{1,2}|2[0-4]\\d|25[0-5])){2}" +
    "(?:\\.(?:[1-9]\\d?|1\\d\\d|2[0-4]\\d|25[0-4]))" +
    "|" +
    // host name
    "(?:(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)" +
    // domain name
    "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)*" +
    // TLD identifier
    "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))" +
    // TLD may end with dot
    "\\.?" +
    ")" +
    // port number
    "(?::\\d{2,5})?" +
    // resource path
    "(?:[/?#]\\S*)?" +
    "$", "i"
);

var isLocalAddressOrRootUrlMatcher = new RegExp(
    "^" +
    "(?:" +
    // protocol identifier
    "(?:(?:https?|ftp)://)" +
    // user:pass authentication
    "(?:\\S+(?::\\S*)?@)?" +
    // host name
    "(?:(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)" +
    // domain name
    "(?:\\.(?:[a-z\\u00a1-\\uffff0-9]-*)*[a-z\\u00a1-\\uffff0-9]+)*" +
    // TLD identifier
    "(?:\\.(?:[a-z\\u00a1-\\uffff]{2,}))?" +
    // TLD may end with dot
    "\\.?" +
    // port number
    "(?::\\d{2,5})?" +
    ")?" +
    // resource path
    "(?:[/# | \./]\\S*)?" +
    "$", "i"
);

module.exports = function (urlCandidate, includeLocalAdressesAndRootUrls) {

    var isValidUrl = false;

    if (includeLocalAdressesAndRootUrls) {
        isValidUrl = isLocalAddressOrRootUrlMatcher.test(urlCandidate);
    }

    return isValidUrl || isPublicUrlMatcher.test(urlCandidate);
};
{'fraction_non_alphanumeric': 0.28663479526086055, 'fraction_numerical': 0.05695281646227396, 'mean_word_length': 1.7639287765651923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459187', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1912, 'n_words': 262}