<commit_before>/// <reference path="_definitions.d.ts" />

var encode: (str: string) => string,
    decode: (str: string) => string;

if (window.btoa) {
    encode = function (text: string): string {
        return window.btoa(unescape(encodeURIComponent(text)));
    };
    decode = function (text: string): string {
        return decodeURIComponent(escape(window.atob(text)));
    };
}
else {
    var _codex = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",
        _utf8_encode = function (text: string): string {
            text = text.replace(/\r\n/g, "\n");
            var result = "";
            for (var n = 0; n < text.length; n++) {
                var c = text.charCodeAt(n);

                if (c < 128) {
                    result += String.fromCharCode(c);
                }
                else if ((c > 127) && (c < 2048)) {
                    result += String.fromCharCode((c >> 6) | 192);
                    result += String.fromCharCode((c & 63) | 128);
                }
                else {
                    result += String.fromCharCode((c >> 12) | 224);
                    result += String.fromCharCode(((c >> 6) & 63) | 128);
                    result += String.fromCharCode((c & 63) | 128);
                }
            }

            return result;
        },
        _utf8_decode = function (text: string): string {
            var result = "",
                i = 0,
                c, c1, c2, c3,
                c = c1 = c2 = 0;

            while (i < text.length) {
                c = text.charCodeAt(i);

                if (c < 128) {
                    result += String.fromCharCode(c);
                    i++;
                }
                else if ((c > 191) && (c < 224)) {
                    c2 = text.charCodeAt(i + 1);
                    result += String.fromCharCode(((c & 31) << 6) | (c2 & 63));
                    i += 2;
                }
                else {
                    c2 = text.charCodeAt(i + 1);
                    c3 = text.charCodeAt(i + 2);
                    result += String.fromCharCode(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
                    i += 3;
                }
            }
<commit_msg>Fix little issue in base64 module<commit_after>/// <reference path="_definitions.d.ts" />

var encode: (str: string) => string,
    decode: (str: string) => string;

if (window.btoa) {
    encode = function (text: string): string {
        return window.btoa(unescape(encodeURIComponent(text)));
    };
    decode = function (text: string): string {
        return decodeURIComponent(escape(window.atob(text)));
    };
}
else {
    var _codex = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=",
        _utf8_encode = function (text: string): string {
            text = text.replace(/\r\n/g, "\n");
            var result = "";
            for (var n = 0; n < text.length; n++) {
                var c = text.charCodeAt(n);

                if (c < 128) {
                    result += String.fromCharCode(c);
                }
                else if ((c > 127) && (c < 2048)) {
                    result += String.fromCharCode((c >> 6) | 192);
                    result += String.fromCharCode((c & 63) | 128);
                }
                else {
                    result += String.fromCharCode((c >> 12) | 224);
                    result += String.fromCharCode(((c >> 6) & 63) | 128);
                    result += String.fromCharCode((c & 63) | 128);
                }
            }

            return result;
        },
        _utf8_decode = function (text: string): string {
            var result = "",
                i = 0,
                c, c1, c2, c3;

            c = c1 = c2 = 0;

            while (i < text.length) {
                c = text.charCodeAt(i);

                if (c < 128) {
                    result += String.fromCharCode(c);
                    i++;
                }
                else if ((c > 191) && (c < 224)) {
                    c2 = text.charCodeAt(i + 1);
                    result += String.fromCharCode(((c & 31) << 6) | (c2 & 63));
                    i += 2;
                }
                else {
                    c2 = text.charCodeAt(i + 1);
                    c3 = text.charCodeAt(i + 2);
                    result += String.fromCharCode(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63));
                    i += 3;
                }
            }

{'fraction_non_alphanumeric': 0.143849428635447, 'fraction_numerical': 0.04033161550526552, 'mean_word_length': 1.1957697983275948, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983762', 'n_tokens_mistral': 1504, 'n_tokens_neox': 1288, 'n_words': 305}