<commit_before>/* This software is licensed under a BSD license; see the LICENSE file for details. */

function createJsHTMLTag(s, namesHash) {
    m = s.match(/^\s*(\w+)(?::(\w+))?$/);
    if (! m)
        assert(false, "Badly formatted jsHTML tag: '" + s + "'.");
    var elem = document.createElement(m[1]);
    var n = m[2];
    if (namesHash != null && n)
        namesHash[n] = elem;
    return elem;
}

function jsHTML(html, namesHash) {
    if (typeof(html) == "string") {
        return document.createTextNode(html);
    }

    assert_is_arraylike(html, "Bad jsHTML.");
    assert(html.length > 0, "Bad jsHTML.");
    var elem;
    if (typeof(html[0]) == "string") {
        if (! (html[0].charAt(0) == "&" && html[0].charAt(html[0].length - 1) == (";"))) {
            elem = createJsHTMLTag(html[0], namesHash);
        }
        else {
            assert(html.length == 1, "Entity cannot have children.");
            // Bit of a hack -- haven't been able to find a proper DOM way to do this.
            if (! html[0].match(/^&\s*(?:(?:[a-zA-Z]+)|(?:#\d+))\s*;$/)) {
                assert(false, "Bad entity: '" + html + "'.");
            }
            elem = document.createElement("span");
            elem.innerHTML = html[0];
        }
    }
    else {
        assert_is_arraylike(html[0], "Bad jsHTML.");
        assert(html[0].length == 1 || html[0].length == 2, "Bad jsHTML.");
        elem = createJsHTMLTag(html[0][0], namesHash);
        if (html[0].length == 2) {
            for (var k in html[0][1]) {
                // Is this setting an attribute or a DOM object key?
                if (! (k.charAt(0) == "@")) {
                    elem.setAttribute(k, html[0][1][k]);
                    if (k == "class")
                        elem.setAttribute("className", html[0][1][k]);
                }
                else {
                    var m = k.match(/^@\s*(?:([a-zA-Z_]\w*)\.)*([a-zA-Z_]\w*)\s*$/);
                    assert(m, "Badly formatted DOM attribute selector in jsHTML.");
                    var initials = m[1] ? m[1].split(".") : [];
                    var lastObject = elem;
                    for (var i = 0; i < initials.length; ++i)
                        lastObject = lastObject[initials[i]];
                    lastObject[m[2]] = html[0][1][k];
                }
            }
        }
    }

    for (var i = 1; i < html.length; ++i) {
        elem.appendChild(jsHTML(html[i], namesHash));
        // Should we add a space separator?
        if (i < html.length - 1                                       &&
            (! (typeof(html[i]) == "object" &&
                typeof(html[i][0]) == "string" &&
                html[i][0].charAt(0) == "&" &&
                html[i][0].charAt(html[i][0].length - 1) == ";"))     &&
            (! (i < html.length - 2 &&
               typeof(html[i + 1]) == "object" &&
               typeof(html[i + 1][0]) == "string" &&
               html[i + 1][0].charAt(0) == "&" &&
               html[i + 1][0].charAt(html[i + 1][0].length - 1) == ";"))) {
            elem.appendChild(document.createTextNode(" "));
        } 
    }

    return elem;
}
function jsHTMLWithNames(names, html) { return jsHTML(html, names); }

function htmlCodeToDOM(html) {
    if (typeof(html) == "string") {
        var d = document.createElement("div");
        d.innerHTML = html;
        return d;
    }
    else {
        return jsHTML(html);
    }
}

// Taken from http://aymanh.com/9-javascript-tips-you-may-not-know
function AssertException(message) { this.message = message; }
AssertException.prototype.toString = function () {
    return 'AssertException: ' + this.message;
}
function assert(exp, message) {
    if (! exp) {
        if (message)
            alert("ERROR: " + message);
        throw new AssertException(message);
    }
}

function assert_is_arraylike(expr, message) {
    assert((! (expr == null)) && typeof(expr.length) == "number", message)
}

function assert_is_dict(expr, message) {
    for (var _ in expr) {
        return;
    }
    assert(false, message);
}

function stringStartsWith(k, s) {
    // Avoid searching through the whole string in cases where
    // it's not necessary.
    if (s.length == 0 && k.length == 0)
        return true;
    else if (s.charAt(0) != k.charAt(0))
        return false;
    else
        return s.indexOf(k) == 0;
}
function stringEndsWith(k, s) {
    // Avoid searching through the whole string in cases where
    // it's not necessary.
    if (s.length == 0 && k.length == 0)
        return true;
    else if (s.charAt(s.length - 1) != k.charAt(k.length - 1))
        return false
    else {
        var i = s.indexOf(k);
        return k != -1 && i == s.length - k.length;
    }
}

// URL encode chars in a string which will screw up a CSV file (we leave spaces in as it gets very ugly otherwise).
function csv_url_encode(s) {
    var insertions = [];
    for (var i = 0; i < s.length; ++i) {
        if (s.charCodeAt(i) < 32 || s.charCodeAt(i) == 127 || s.charAt(i) == ",") {
            var sr = s.charCodeAt(i).toString(16);
            if (sr.length == 1) sr = "0" + sr;
            insertions.push([i, "%" + sr])
        }
    }
    var slices = [];
    var lastIndex = 0;
    for (var j = 0; j < insertions.length; ++j) {
        var ins = insertions[j];
        if (lastIndex != ins[0])
            slices.push(s.slice(lastIndex, ins[0]));
        slices.push(ins[1]);
        lastIndex = ins[0] + 1;
    }
    
    var js = slices.join("");
    if (insertions.length > 0 && insertions[insertions.length-1][0] < s.length)
        js += s.slice(insertions[insertions.length-1][0] + 1, s.length);
    return js;
}

function getXMLHttpRequest()
{
    if (window.XMLHttpRequest) {
        return new XMLHttpRequest()
    }
    else if (window.ActiveXObject) {
        return new ActiveXObject("Microsoft.XMLHTTP")
    }
    else {
        return null;<commit_msg>Minor url encoding update.<commit_after>/* This software is licensed under a BSD license; see the LICENSE file for details. */

function createJsHTMLTag(s, namesHash) {
    m = s.match(/^\s*(\w+)(?::(\w+))?$/);
    if (! m)
        assert(false, "Badly formatted jsHTML tag: '" + s + "'.");
    var elem = document.createElement(m[1]);
    var n = m[2];
    if (namesHash != null && n)
        namesHash[n] = elem;
    return elem;
}

function jsHTML(html, namesHash) {
    if (typeof(html) == "string") {
        return document.createTextNode(html);
    }

    assert_is_arraylike(html, "Bad jsHTML.");
    assert(html.length > 0, "Bad jsHTML.");
    var elem;
    if (typeof(html[0]) == "string") {
        if (! (html[0].charAt(0) == "&" && html[0].charAt(html[0].length - 1) == (";"))) {
            elem = createJsHTMLTag(html[0], namesHash);
        }
        else {
            assert(html.length == 1, "Entity cannot have children.");
            // Bit of a hack -- haven't been able to find a proper DOM way to do this.
            if (! html[0].match(/^&\s*(?:(?:[a-zA-Z]+)|(?:#\d+))\s*;$/)) {
                assert(false, "Bad entity: '" + html + "'.");
            }
            elem = document.createElement("span");
            elem.innerHTML = html[0];
        }
    }
    else {
        assert_is_arraylike(html[0], "Bad jsHTML.");
        assert(html[0].length == 1 || html[0].length == 2, "Bad jsHTML.");
        elem = createJsHTMLTag(html[0][0], namesHash);
        if (html[0].length == 2) {
            for (var k in html[0][1]) {
                // Is this setting an attribute or a DOM object key?
                if (! (k.charAt(0) == "@")) {
                    elem.setAttribute(k, html[0][1][k]);
                    if (k == "class")
                        elem.setAttribute("className", html[0][1][k]);
                }
                else {
                    var m = k.match(/^@\s*(?:([a-zA-Z_]\w*)\.)*([a-zA-Z_]\w*)\s*$/);
                    assert(m, "Badly formatted DOM attribute selector in jsHTML.");
                    var initials = m[1] ? m[1].split(".") : [];
                    var lastObject = elem;
                    for (var i = 0; i < initials.length; ++i)
                        lastObject = lastObject[initials[i]];
                    lastObject[m[2]] = html[0][1][k];
                }
            }
        }
    }

    for (var i = 1; i < html.length; ++i) {
        elem.appendChild(jsHTML(html[i], namesHash));
        // Should we add a space separator?
        if (i < html.length - 1                                       &&
            (! (typeof(html[i]) == "object" &&
                typeof(html[i][0]) == "string" &&
                html[i][0].charAt(0) == "&" &&
                html[i][0].charAt(html[i][0].length - 1) == ";"))     &&
            (! (i < html.length - 2 &&
               typeof(html[i + 1]) == "object" &&
               typeof(html[i + 1][0]) == "string" &&
               html[i + 1][0].charAt(0) == "&" &&
               html[i + 1][0].charAt(html[i + 1][0].length - 1) == ";"))) {
            elem.appendChild(document.createTextNode(" "));
        } 
    }

    return elem;
}
function jsHTMLWithNames(names, html) { return jsHTML(html, names); }

function htmlCodeToDOM(html) {
    if (typeof(html) == "string") {
        var d = document.createElement("div");
        d.innerHTML = html;
        return d;
    }
    else {
        return jsHTML(html);
    }
}

// Taken from http://aymanh.com/9-javascript-tips-you-may-not-know
function AssertException(message) { this.message = message; }
AssertException.prototype.toString = function () {
    return 'AssertException: ' + this.message;
}
function assert(exp, message) {
    if (! exp) {
        if (message)
            alert("ERROR: " + message);
        throw new AssertException(message);
    }
}

function assert_is_arraylike(expr, message) {
    assert((! (expr == null)) && typeof(expr.length) == "number", message)
}

function assert_is_dict(expr, message) {
    for (var _ in expr) {
        return;
    }
    assert(false, message);
}

function stringStartsWith(k, s) {
    // Avoid searching through the whole string in cases where
    // it's not necessary.
    if (s.length == 0 && k.length == 0)
        return true;
    else if (s.charAt(0) != k.charAt(0))
        return false;
    else
        return s.indexOf(k) == 0;
}
function stringEndsWith(k, s) {
    // Avoid searching through the whole string in cases where
    // it's not necessary.
    if (s.length == 0 && k.length == 0)
        return true;
    else if (s.charAt(s.length - 1) != k.charAt(k.length - 1))
        return false
    else {
        var i = s.indexOf(k);
        return k != -1 && i == s.length - k.length;
    }
}

function custom_url_encode(s, specials) {
    function is_special(c) {
        for (var i = 0; i < specials.length; ++i) {
            if (specials.charAt(i) == c.charAt(0))
                return true;
        }
        return false;
    }

    var insertions = [];
    for (var i = 0; i < s.length; ++i) {
        if (s.charCodeAt(i) < 32 || s.charCodeAt(i) == 127 || (specials && is_special(s.charAt(i)))) {
            var sr = s.charCodeAt(i).toString(16);
            if (sr.length == 1) sr = "0" + sr;
            insertions.push([i, "%" + sr])
        }
    }
    var slices = [];
    var lastIndex = 0;
    for (var j = 0; j < insertions.length; ++j) {
        var ins = insertions[j];
        if (lastIndex != ins[0])
            slices.push(s.slice(lastIndex, ins[0]));
        slices.push(ins[1]);
        lastIndex = ins[0] + 1;
    }
    
    var js = slices.join("");
    if (insertions.length > 0 && insertions[insertions.length-1][0] < s.length)
        js += s.slice(insertions[insertions.length-1][0] + 1, s.length);
    return js;
}

// URL encode chars in a string which will screw up a CSV file (we leave spaces in as it gets very ugly otherwise).
function csv_url_encode(s) {
    return custom_url_encode(s, ",");
}

alert(csv_url_encode("fo,of"))

function getXMLHttpRequest()
{
    if (window.XMLHttpRequest) {
        return new XMLHttpRequest()
    }
    else if (window.ActiveXObject) {
        return new ActiveXObject("Microsoft.XMLHTTP")
    }
    else {
        return null;
{'fraction_non_alphanumeric': 0.17277873492493231, 'fraction_numerical': 0.014767413241447206, 'mean_word_length': 1.8879412461502014, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5722354', 'n_tokens_mistral': 4061, 'n_tokens_neox': 3892, 'n_words': 941}