<commit_before>/*
  
  BiSON.js
  Copyright (c) 2010 <NAME>.
  
  All rights reserved.
  
  BiSON.js is free software: you can redistribute it and/or
  modify it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  BiSON.js is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License along with
  BiSON.js. If not, see <http://www.gnu.org/licenses/>.
  
*/

(function() {
var floor = Math.floor, round = Math.round, ceil = Math.ceil, chr = String.fromCharCode;
var tok = [];
for (var i = 0; i < 256; i++) {
    tok.push(chr(i));
}

var enc = '';
function _encode(data) {
    if (typeof data === 'number') {
        
        // Floats
        var add = 0, f = floor(data);
        if (f !== data) {
            var m = data > 0 ? f : ceil(data);
            var r = round((data - m) * 100);
            if (m < 0 || r < 0) {
                m = 0 - m;
                r = 0 - r;
                add = 1;
            }
            
            if (m < 256) {
                if (m === 0) {
                    enc += tok[13 + add] + tok[r + 128];
                
                } else {
                    enc += tok[13 + add] + tok[r] + tok[m];
                }
            
            } else if (m < 65536) {
                enc += tok[15 + add] + tok[m >> 8 & 0xff]
                                     + tok[m & 0xff] + tok[r];
            
            } else if (m < 2147483648) {
                enc += tok[17 + add] + tok[m >> 24 & 0xff]
                                     + tok[m >> 16 & 0xff]
                                     + tok[m >> 8 & 0xff]
                                     + tok[m & 0xff] + tok[r];
            
            } else if (m === 2147483648) {
                enc += tok[18] + tok[0] + tok[0] + tok[0] + tok[0] + tok[r];
            }
        
        // Fixed
        } else {
            if (data <= 0) {
                data = 0 - data;
                add = 1;
            
            } else {
                data--;
            }
            
            if (data < 112) {
                enc += tok[25 + data + add * 112];
            
            } else if (data < 256) {
                enc += tok[1 + add] + tok[data];
            
            } else if (data < 65536) {
                enc += tok[3 + add] + tok[data >> 8 & 0xff]
                                    + tok[data & 0xff];
            
            } else if (data < 2147483648) {
                enc += tok[5 + add] + tok[data >> 24 & 0xff]
                                    + tok[data >> 16 & 0xff]
                                    + tok[data >> 8 & 0xff]
                                    + tok[data & 0xff];
            
            } else {
                enc += tok[1 + add] + tok[0];
            }
        }
    
    // Strings
    } else if (typeof data === 'string') {
        enc += tok[7] + data + tok[0];
    
    // Booleans
    } else if (typeof data === 'boolean') {
        enc += tok[data ? 19 : 20];
    
    // Null
    } else if (data === null) {
        enc += tok[0];
    
    // Objects / Arrays
    } else if (typeof data === 'object') {
        if (data instanceof Array) {
            enc += tok[8];
            for (var i = 0, l = data.length; i < l; i++) {
                _encode(data[i]);
            }
            enc += tok[9];
        
        } else {
            enc += tok[10];
            for (var e in data) {
                enc += tok[25 + e.length] + e;
                _encode(data[e]);
            }
            enc += tok[11];
        }
    }
}

function encode(data) {
    enc = '';
    _encode(data);
    return enc;
}

function decode(data) {
    var p = 0, l = data.length;
    var s = [], d = null, f = null, t = 0, size = 0, i = -1;
    var dict = false, set = false;
    var str = '', k ='', e = '';
    while (p < l) {
        t = data.charCodeAt(p++);
        f = s[i];
        
        // Keys
        if (dict && set && t > 24) {
            k = data.substring(p, p + t - 25);
            p += t - 25;
            set = false;
        
        // Array / Objects
        } else if (t === 8 || t === 10) {
            var a = t === 8 ? new Array() : new Object();
            set = dict = t === 10;
            d !== null ? f instanceof Array ? f.push(a) : f[k] = a : d = a;
            s.push(a);
            i++;
        
        } else if (t === 11 || t === 9) {
            s.pop(i--);
            set = dict = !(s[i] instanceof Array);
        
        // Fixed
        } else if (t > 24) {
            t = t - 25;
            t = t > 111 ? (0 - t + 112) : t + 1;
            f instanceof Array ? f.push(t) : f[k] = t;
            set = true;
        
        } else if (t > 0 && t < 7) {
            size = floor((t - 1) / 2);
            var value = 0;
            if (size === 0) {
                value = data.charCodeAt(p);
                p++;
            
            } else if (size === 1) {
                value = (data.charCodeAt(p) << 8) + data.charCodeAt(p + 1);
                p += 2;
            
            } else if (size === 2) {<commit_msg>Guard against as possible problem with nulls.<commit_after>/*
  
  BiSON.js
  Copyright (c) 2010 <NAME>.
  
  All rights reserved.
  
  BiSON.js is free software: you can redistribute it and/or
  modify it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  BiSON.js is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License along with
  BiSON.js. If not, see <http://www.gnu.org/licenses/>.
  
*/

(function(undefined) {
var floor = Math.floor, round = Math.round, ceil = Math.ceil, chr = String.fromCharCode;
var tok = [];
for (var i = 0; i < 256; i++) {
    tok.push(chr(i));
}

var enc = '';
function _encode(data) {
    if (typeof data === 'number') {
        
        // Floats
        var add = 0, f = floor(data);
        if (f !== data) {
            var m = data > 0 ? f : ceil(data);
            var r = round((data - m) * 100);
            if (m < 0 || r < 0) {
                m = 0 - m;
                r = 0 - r;
                add = 1;
            }
            
            if (m < 256) {
                if (m === 0) {
                    enc += tok[13 + add] + tok[r + 128];
                
                } else {
                    enc += tok[13 + add] + tok[r] + tok[m];
                }
            
            } else if (m < 65536) {
                enc += tok[15 + add] + tok[m >> 8 & 0xff]
                                     + tok[m & 0xff] + tok[r];
            
            } else if (m < 2147483648) {
                enc += tok[17 + add] + tok[m >> 24 & 0xff]
                                     + tok[m >> 16 & 0xff]
                                     + tok[m >> 8 & 0xff]
                                     + tok[m & 0xff] + tok[r];
            
            } else if (m === 2147483648) {
                enc += tok[18] + tok[0] + tok[0] + tok[0] + tok[0] + tok[r];
            }
        
        // Fixed
        } else {
            if (data <= 0) {
                data = 0 - data;
                add = 1;
            
            } else {
                data--;
            }
            
            if (data < 112) {
                enc += tok[25 + data + add * 112];
            
            } else if (data < 256) {
                enc += tok[1 + add] + tok[data];
            
            } else if (data < 65536) {
                enc += tok[3 + add] + tok[data >> 8 & 0xff]
                                    + tok[data & 0xff];
            
            } else if (data < 2147483648) {
                enc += tok[5 + add] + tok[data >> 24 & 0xff]
                                    + tok[data >> 16 & 0xff]
                                    + tok[data >> 8 & 0xff]
                                    + tok[data & 0xff];
            
            } else {
                enc += tok[1 + add] + tok[0];
            }
        }
    
    // Strings
    } else if (typeof data === 'string') {
        enc += tok[7] + data + tok[0];
    
    // Booleans
    } else if (typeof data === 'boolean') {
        enc += tok[data ? 19 : 20];
    
    // Null
    } else if (data === null) {
        enc += tok[0];
    
    // Objects / Arrays
    } else if (typeof data === 'object') {
        if (data instanceof Array) {
            enc += tok[8];
            for (var i = 0, l = data.length; i < l; i++) {
                _encode(data[i]);
            }
            enc += tok[9];
        
        } else {
            enc += tok[10];
            for (var e in data) {
                enc += tok[25 + e.length] + e;
                _encode(data[e]);
            }
            enc += tok[11];
        }
    }
}

function encode(data) {
    enc = '';
    _encode(data);
    return enc;
}

function decode(data) {
    var p = 0, l = data.length;
    var s = [], d = undefined, f = null, t = 0, size = 0, i = -1;
    var dict = false, set = false;
    var str = '', k ='', e = '';
    while (p < l) {
        t = data.charCodeAt(p++);
        f = s[i];
        
        // Keys
        if (dict && set && t > 24) {
            k = data.substring(p, p + t - 25);
            p += t - 25;
            set = false;
        
        // Array / Objects
        } else if (t === 8 || t === 10) {
            var a = t === 8 ? new Array() : new Object();
            set = dict = t === 10;
            d !== undefined ? f instanceof Array ? f.push(a) : f[k] = a : d = a;
            s.push(a);
            i++;
        
        } else if (t === 11 || t === 9) {
            s.pop(i--);
            set = dict = !(s[i] instanceof Array);
        
        // Fixed
        } else if (t > 24) {
            t = t - 25;
            t = t > 111 ? (0 - t + 112) : t + 1;
            f instanceof Array ? f.push(t) : f[k] = t;
            set = true;
        
        } else if (t > 0 && t < 7) {
            size = floor((t - 1) / 2);
            var value = 0;
            if (size === 0) {
                value = data.charCodeAt(p);
                p++;
            
            } else if (size === 1) {
                value = (data.charCodeAt(p) << 8) + data.charCodeAt(p + 1);
                p += 2;
            
            } else if (size === 2) {
{'fraction_non_alphanumeric': 0.14075029956678034, 'fraction_numerical': 0.03336713061111623, 'mean_word_length': 1.0070292267850536, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1228513', 'n_tokens_mistral': 3612, 'n_tokens_neox': 3245, 'n_words': 1036}