<commit_before>// Polyfill for networkFetch() API. Does not handle cross-origin
// requests, because it uses XHR to emulate.
if (!('SameOriginResponse' in this))
    SameOriginResponse = function() { };

function networkFetch(urlOrRequest) {
    return new Promise(function(resolver) {
        var url;
        var method = 'GET';
        var body;
        if (typeof urlOrRequest == "string") {
            url = urlOrRequest;
        } else {
            url = urlOrRequest.url;
            method = urlOrRequest.method || method;
            body = urlOrRequest.body;
        }
        var xhr = new XMLHttpRequest();
        xhr.open(method, url, true);
        if (body)
            xhr.send(body);
        else
            xhr.send();
        xhr.onreadystatechange = function() {
            if (xhr.readyState == 4) {
                try {
                    var response = new SameOriginResponse();
                    // need to .reject based on statusCode?
                    response.statusCode = xhr.status;<commit_msg>make sure to use array buffers so we can respond with binary data<commit_after>// Polyfill for networkFetch() API. Does not handle cross-origin
// requests, because it uses XHR to emulate.
if (!('SameOriginResponse' in this))
    SameOriginResponse = function() { };

function networkFetch(urlOrRequest) {
    return new Promise(function(resolver) {
        var url;
        var method = 'GET';
        var body;
        if (typeof urlOrRequest == "string") {
            url = urlOrRequest;
        } else {
            url = urlOrRequest.url;
            method = urlOrRequest.method || method;
            body = urlOrRequest.body;
        }
        var xhr = new XMLHttpRequest();
        xhr.responseType = "arraybuffer";
        xhr.open(method, url, true);
        if (body)
            xhr.send(body);
        else
            xhr.send();
        xhr.onreadystatechange = function() {
            if (xhr.readyState == 4) {
                try {
                    var response = new SameOriginResponse();
                    // need to .reject based on statusCode?
                    response.statusCode = xhr.status;
{'fraction_non_alphanumeric': 0.10037002775208141, 'fraction_numerical': 0.0009250693802035153, 'mean_word_length': 1.7554140127388536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751431', 'n_tokens_mistral': 581, 'n_tokens_neox': 567, 'n_words': 169}