<commit_before>//
//  Author: <NAME> <EMAIL>
//  backendjs 2018
//

const util = require('util');
const bkutils = require('bkjs-utils');
const lib = require(__dirname + '/lib');

// Perform match on a regexp for a string and returns matched value, if no index is specified returns item 1,
// using index 0 retutns the whole matched string.
// This is a simple case for oneshot match and only single matched element.
// If the `rx`` object has the 'g' flag the result will be all matches in an array.
lib.matchRegexp = function(str, rx, index)
{
    if (typeof str != "string" || !rx || !util.isRegExp(rx)) return null;
    var d = str.match(rx);
    return d ? /g/.test(rx.flags) ? d : d[typeof index == "number" ? index : 1] : null;
}

// Perform match on a regexp and return all matches in an array, if no index is specified returns item 1
lib.matchAllRegexp = function(str, rx, index)
{
    if (typeof str != "string" || !rx || !util.isRegExp(rx)) return [];
    var d, rc = [];
    while ((d = rx.exec(str)) !== null) {
        rc.push(d[typeof index == "number" ? index : 1]);
    }
    return rc;
}

// Perform test on a regexp for a string and returns true only if matched.
lib.testRegexp = function(str, rx)
{
    return rx && util.isRegExp(rx) ? rx.test(str) : false;
}

// Run test on a regexpObj
lib.testRegexpObj = function(str, rx)
{
    return rx && util.isRegExp(rx.rx) ? rx.rx.test(str) : false;
}

// Safe version of replace for strings, always returns a string, if `val` is not provided performs
// removal of the matched patterns
lib.replaceRegexp = function(str, rx, val)
{
    if (typeof str != "string") return "";
    if (!util.isRegExp(rx)) return str;
    return str.replace(rx, val || "");
}

// Remove all whitespace from the begining and end of the given string, if an array with characters is not given then it trims all whitespace
lib.strTrim = function(str, chars)
{
    if (typeof str != "string" || !str) return "";
    var rx;
    if (typeof chars == "string" && chars) {
        rx = new RegExp("(^[" + chars + "]+)|([" + chars + "]+$)", "gi");
    } else {
        if (!this._whitespace) {
            this._whitespace = new RegExp("(^[" + this.whitespace + "]+)|([" + this.whitespace + "]+$)", "gi");
        }
        rx = this._whitespace;
    }
    return str.replace(rx, "");
}

// Split string into array, ignore empty items,
// - `sep` is an RegExp to use as a separator instead of default  pattern `[,\|]`,
// - `options` is an object with the same properties as for the `toParams`, `datatype' will be used with
//   `lib.toValue` to convert the value for each item
// - `options.keepempty` - will preserve empty items, by default empty is ignored
//
// If `str` is an array and type is not specified then all non-string items will be returned as is.
lib.strSplit = function(str, sep, options)
{
    if (!str) return [];
    options = options || this.empty;
    return (Array.isArray(str) ? str : (typeof str == "string" ? str : String(str)).split(sep || this.rxSplit)).
            map((x) => {
                x = options.datatype ? lib.toValue(x, options.datatype) : typeof x == "string" ? x.trim() : x;
                if (typeof x == "string") {
                    if (options.regexp && !options.regexp.test(x)) return "";
                    if (options.lower) x = x.toLowerCase();
                    if (options.upper) x = x.toUpperCase();
                    if (options.strip) x = x.replace(options.strip, "");
                    if (options.camel) x = lib.toCamel(x, options);
                    if (options.cap) x = lib.toTitle(x);
                    if (options.trunc > 0) x = x.substr(0, options.trunc);
                }
                return x;
            }).
            filter((x) => (options.keepempty || typeof x == "string" ? x.length : 1));
}

// Split as above but keep only unique items, case-insensitive
lib.strSplitUnique = function(str, sep, options)
{
    var rc = [];
    var typed = options && options.datatype != "undefined";
    this.strSplit(str, sep, options).forEach(function(x) {
        if (!rc.some(function(y) {
            return typed || !(typeof x == "string" && typeof y == "string") ? x == y : x.toLowerCase() == y.toLowerCase()
        })) rc.push(x);
    });
    return rc;
}<commit_msg>more optimized strSplit<commit_after>//
//  Author: <NAME> <EMAIL>
//  backendjs 2018
//

const util = require('util');
const bkutils = require('bkjs-utils');
const lib = require(__dirname + '/lib');

// Perform match on a regexp for a string and returns matched value, if no index is specified returns item 1,
// using index 0 retutns the whole matched string.
// This is a simple case for oneshot match and only single matched element.
// If the `rx`` object has the 'g' flag the result will be all matches in an array.
lib.matchRegexp = function(str, rx, index)
{
    if (typeof str != "string" || !rx || !util.isRegExp(rx)) return null;
    var d = str.match(rx);
    return d ? /g/.test(rx.flags) ? d : d[typeof index == "number" ? index : 1] : null;
}

// Perform match on a regexp and return all matches in an array, if no index is specified returns item 1
lib.matchAllRegexp = function(str, rx, index)
{
    if (typeof str != "string" || !rx || !util.isRegExp(rx)) return [];
    var d, rc = [];
    while ((d = rx.exec(str)) !== null) {
        rc.push(d[typeof index == "number" ? index : 1]);
    }
    return rc;
}

// Perform test on a regexp for a string and returns true only if matched.
lib.testRegexp = function(str, rx)
{
    return rx && util.isRegExp(rx) ? rx.test(str) : false;
}

// Run test on a regexpObj
lib.testRegexpObj = function(str, rx)
{
    return rx && util.isRegExp(rx.rx) ? rx.rx.test(str) : false;
}

// Safe version of replace for strings, always returns a string, if `val` is not provided performs
// removal of the matched patterns
lib.replaceRegexp = function(str, rx, val)
{
    if (typeof str != "string") return "";
    if (!util.isRegExp(rx)) return str;
    return str.replace(rx, val || "");
}

// Remove all whitespace from the begining and end of the given string, if an array with characters is not given then it trims all whitespace
lib.strTrim = function(str, chars)
{
    if (typeof str != "string" || !str) return "";
    var rx;
    if (typeof chars == "string" && chars) {
        rx = new RegExp("(^[" + chars + "]+)|([" + chars + "]+$)", "gi");
    } else {
        if (!this._whitespace) {
            this._whitespace = new RegExp("(^[" + this.whitespace + "]+)|([" + this.whitespace + "]+$)", "gi");
        }
        rx = this._whitespace;
    }
    return str.replace(rx, "");
}

// Split string into array, ignore empty items,
// - `sep` is an RegExp to use as a separator instead of default  pattern `[,\|]`,
// - `options` is an object with the same properties as for the `toParams`, `datatype' will be used with
//   `lib.toValue` to convert the value for each item
// - `options.keepempty` - will preserve empty items, by default empty strings are ignored
// - `options.notrim` - will skip trimming strings, trim is the default
//
// If `str` is an array and type is not specified then all non-string items will be returned as is.
lib.strSplit = function(str, sep, options)
{
    if (!str) return [];
    options = options || lib.empty;
    var list = (Array.isArray(str) ? str : (typeof str == "string" ? str : String(str)).split(sep || this.rxSplit)), len = list.length;
    if (!len) return list;

    var rc = [], keys = Object.keys(options), v;
    for (let i = 0; i < len; ++i) {
        v = list[i];
        if (options.datatype) v = lib.toValue(v, options.datatype);
        if (typeof v != "string") {
            rc.push(v);
            continue;
        }
        if (!options.notrim) v = v.trim();

        for (let k = 0; k < keys.length; ++k) {
            switch (keys[k]) {
            case "regexp":
                if (!options.regexp.test(v)) v = "";
                break;
            case "lower":
                v = v.toLowerCase();
                break;
            case "upper":
                v = v.toUpperCase();
                break;
            case "strip":
                v = v.replace(options.strip, "");
                break;
            case "camel":
                v = lib.toCamel(v, options);
                break;
            case "cap":
                v = lib.toTitle(v);
                break;
            case "trunc":
                v = v.substr(0, options.trunc);
                break;
            }
        }
        if (!v.length && !options.keepempty) continue;
        rc.push(v);
    }
    return rc;
}

// Split as above but keep only unique items, case-insensitive
lib.strSplitUnique = function(str, sep, options)
{
    var rc = [];
    var typed = options && options.datatype != "undefined";
    this.strSplit(str, sep, options).forEach((x) => {
        if (!rc.some((y) => (typed || !(typeof x == "string" && typeof y == "string") ? x == y : x.toLowerCase() == y.toLowerCase()))) rc.push(x);
    });
    return rc;
}
{'fraction_non_alphanumeric': 0.15198847900742218, 'fraction_numerical': 0.0026586905948820204, 'mean_word_length': 2.4132325141776936, 'pattern_counts': {'":': 7, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '55414', 'n_tokens_mistral': 2850, 'n_tokens_neox': 2795, 'n_words': 973}