<commit_before>
const template = {
    add: function (a, b) a + b,
    join: function (c) function (a, b) a + c + b,

    map: function (iter, fn, sep)
    {
        if (iter.length) /* Kludge? */
            iter = util.arrayIter(iter);
        let ret = <></>;
        let n = 0;
        for each (let i in Iterator(iter))
        {
            let val = fn(i);
            if (val == undefined)
                continue;
            if (sep && n++)
                ret += sep;
            ret += val;
        }
        return ret;
    },

    maybeXML: function (xml)
    {
        if (typeof xml == "xml")
            return xml;
        try
        {
            return new XMLList(xml);
        }
        catch (e) {}
        return <>{xml}</>;
    },

    // if "processStrings" is true, any passed strings will be surrounded by " and
    // any line breaks are displayed as \n
    highlight: function (arg, processStrings)
    {
        // some objects like window.JSON or getBrowsers()._browsers need the try/catch
        try
        {
	    switch (arg == null ? "undefined" : typeof arg)
	    {
		case "number":
		    return <span class="hl-Number">{arg}</span>;
		case "string":
                    if (processStrings)
                        arg = <>{util.escapeString(arg)}</>;
                    return <span class="hl-String">{arg}</span>;
                case "boolean":
                    return <span class="hl-Boolean">{arg}</span>;
                case "function":
                    // Vim generally doesn't like /foo*/, because */ looks like a comment terminator.
                    // Using /foo*(:?)/ instead.
                    if (processStrings)
                        return <span class="hl-Function">{String(arg).replace(/\{(.|\n)*(?:)/, "{ ... }")}</span>;
                    return <>{arg}</>;
                case "undefined":
                    return <span class="hl-Null">{arg}</span>;
                case "object":
                    // for java packages value.toString() would crash so badly
                    // that we cannot even try/catch it
                    if (/^\[JavaPackage.*\]$/.test(arg))
                        return <>[JavaPackage]</>;
                    if (processStrings)
                        arg = String(arg).replace("\n", "\\n", "g");
                    return <span class="hl-Object">{arg}</span>;
                default:
                    return <![CDATA[<unknown type>]]>;
            }
        }
        catch (e)
        {
            return<![CDATA[<unknown>]]>;
        }
    },

    highlightFilter: function (str, filter)
    {
        if (typeof str == "xml")
            return str;
        if (str == "")
            return <>{str}</>;
        let lcstr = String(str).toLowerCase();
        let lcfilter = filter.toLowerCase();
        str = String(str).replace(" ", "\u00a0");
        let s = <></>;
        let start = 0;
        let i;
        while ((i = lcstr.indexOf(lcfilter, start)) > -1)
        {
            s += <>{str.substring(start, i)}</>;<commit_msg>Fix completion filter highlighting wrt white space collapsing.<commit_after>
const template = {
    add: function (a, b) a + b,
    join: function (c) function (a, b) a + c + b,

    map: function (iter, fn, sep)
    {
        if (iter.length) /* Kludge? */
            iter = util.arrayIter(iter);
        let ret = <></>;
        let n = 0;
        for each (let i in Iterator(iter))
        {
            let val = fn(i);
            if (val == undefined)
                continue;
            if (sep && n++)
                ret += sep;
            ret += val;
        }
        return ret;
    },

    maybeXML: function (xml)
    {
        if (typeof xml == "xml")
            return xml;
        try
        {
            return new XMLList(xml);
        }
        catch (e) {}
        return <>{xml}</>;
    },

    // if "processStrings" is true, any passed strings will be surrounded by " and
    // any line breaks are displayed as \n
    highlight: function (arg, processStrings)
    {
        // some objects like window.JSON or getBrowsers()._browsers need the try/catch
        try
        {
	    switch (arg == null ? "undefined" : typeof arg)
	    {
		case "number":
		    return <span class="hl-Number">{arg}</span>;
		case "string":
                    if (processStrings)
                        arg = <>{util.escapeString(arg)}</>;
                    return <span class="hl-String">{arg}</span>;
                case "boolean":
                    return <span class="hl-Boolean">{arg}</span>;
                case "function":
                    // Vim generally doesn't like /foo*/, because */ looks like a comment terminator.
                    // Using /foo*(:?)/ instead.
                    if (processStrings)
                        return <span class="hl-Function">{String(arg).replace(/\{(.|\n)*(?:)/, "{ ... }")}</span>;
                    return <>{arg}</>;
                case "undefined":
                    return <span class="hl-Null">{arg}</span>;
                case "object":
                    // for java packages value.toString() would crash so badly
                    // that we cannot even try/catch it
                    if (/^\[JavaPackage.*\]$/.test(arg))
                        return <>[JavaPackage]</>;
                    if (processStrings)
                        arg = String(arg).replace("\n", "\\n", "g");
                    return <span class="hl-Object">{arg}</span>;
                default:
                    return <![CDATA[<unknown type>]]>;
            }
        }
        catch (e)
        {
            return<![CDATA[<unknown>]]>;
        }
    },

    highlightFilter: function (str, filter)
    {
        if (typeof str == "xml")
            return str;
        if (str == "")
            return <>{str}</>;
        let lcstr = String(str).toLowerCase();
        let lcfilter = filter.toLowerCase();
        str = String(str).replace(" ", "\u00a0");
        let s = <></>;
        let start = 0;
        let i;
        while ((i = lcstr.indexOf(lcfilter, start)) > -1)
        {
	    XML.ignoreWhitespace = false;
            s += <>{str.substring(start, i)}</>;
{'fraction_non_alphanumeric': 0.15114405414115373, 'fraction_numerical': 0.0019336126329358686, 'mean_word_length': 1.316044776119403, 'pattern_counts': {'":': 12, '<': 67, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7418270', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1664, 'n_words': 476}