<commit_before>
/* jslint bitwise: true */

var jsEncoder = (function() {

    //   Set strict mode, forced to
    //   fix up sloppy code
    'use strict';
    
    //
    //    Base64 encode/decode functions
    //
    
    //    Base64 character array
    var b64Str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
    
    //
    //    Public
    //    Base64 encode
    //
    var encode64 = function( data ) {
    
        var c1, c2, c3, e1, e2, e3, e4,
            i   = 0,
            out = "",
            len = data.length,
            rem = len % 3;
        
        if( !isArray( data ) ) {
            
            throw new Error( "TypeError - Input must be an array" );
        }
        
        //
        //  Pad with nulls until length is 
        //  a multiple of 3
        //
        if( rem === 1 ) {
            
            data.push( 0, 0 );
            len += 2;
            
        } else if( rem === 2 ) {
            
            data.push( 0 );
            len += 1;
        }             
        
        while( i < len ) {
            
            c1 = data[ i++ ];
            c2 = data[ i++ ];
            c3 = data[ i++ ];
            
            e1 = c1 >> 2;
            e2 = ((c1 << 4) & 48) | (c2 >> 4);
            e3 = ((c2 << 2) & 60) | (c3 >> 6);
            e4 = c3 & 63;
            
            out += ( b64Str.charAt( e1 ) + b64Str.charAt( e2 ) + b64Str.charAt( e3 ) + b64Str.charAt( e4 ) );
        }
        
        //
        //  Replace the last 1 or 2 characters with "=" if the original
        //  data length was not a multiple of 3
        //
        if( rem === 1 ) {
            
            out = out.substring( 0, out.length - 2 ).concat( "==" );
            
        } else if( rem === 2 ) {
            
            out = out.substring( 0, out.length - 1 ).concat( "=" );
        }
        
        return out;    
    };
    
    
    //
    //    Public
    //    Base64 decode
    // 
    var decode64 = function( data ) {
    
        var e1, e2, e3, e4,
            i   = 0,
            len,
            out = [];
        
        //
        //  Make sure input length is a multiple of 4 and only contains
        //  valid characters
        //
        if( typeof data !== 'string' || ( len % 4 !== 0 ) || (/[A-Za-z0-9+=\/]+/).test( data ) )  {
            
            throw new Error( "Input not properly encoded" );            
        }
        <commit_msg>Fixed regex check on base64 decode<commit_after>
/* jslint bitwise: true */

var jsEncoder = (function() {

    //   Set strict mode, forced to
    //   fix up sloppy code
    'use strict';
    
    //
    //    Base64 encode/decode functions
    //
    
    //    Base64 character array
    var b64Str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
    
    //
    //    Public
    //    Base64 encode
    //
    var encode64 = function( data ) {
    
        var c1, c2, c3, e1, e2, e3, e4,
            i   = 0,
            out = "",
            len = data.length,
            rem = len % 3;
        
        if( !isArray( data ) ) {
            
            throw new Error( "TypeError - Input must be an array" );
        }
        
        //
        //  Pad with nulls until length is 
        //  a multiple of 3
        //
        if( rem === 1 ) {
            
            data.push( 0, 0 );
            len += 2;
            
        } else if( rem === 2 ) {
            
            data.push( 0 );
            len += 1;
        }             
        
        while( i < len ) {
            
            c1 = data[ i++ ];
            c2 = data[ i++ ];
            c3 = data[ i++ ];
            
            e1 = c1 >> 2;
            e2 = ((c1 << 4) & 48) | (c2 >> 4);
            e3 = ((c2 << 2) & 60) | (c3 >> 6);
            e4 = c3 & 63;
            
            out += ( b64Str.charAt( e1 ) + b64Str.charAt( e2 ) + b64Str.charAt( e3 ) + b64Str.charAt( e4 ) );
        }
        
        //
        //  Replace the last 1 or 2 characters with "=" if the original
        //  data length was not a multiple of 3
        //
        if( rem === 1 ) {
            
            out = out.substring( 0, out.length - 2 ).concat( "==" );
            
        } else if( rem === 2 ) {
            
            out = out.substring( 0, out.length - 1 ).concat( "=" );
        }
        
        return out;    
    };
    
    
    //
    //    Public
    //    Base64 decode
    // 
    var decode64 = function( data ) {
    
        var e1, e2, e3, e4,
            i   = 0,
            len,
            out = [];
        
        //
        //  Make sure input length is a multiple of 4 and only contains
        //  valid characters
        //
        if( typeof data !== 'string' || ( len % 4 !== 0 ) || !(/[A-Za-z0-9+=\/]+/).test( data ) )  {
            
            throw new Error( "Input not properly encoded" );            
        }
        
{'fraction_non_alphanumeric': 0.13148298217179902, 'fraction_numerical': 0.039303079416531606, 'mean_word_length': 0.9017719568567026, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6382981', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1543, 'n_words': 431}