<commit_before>use std::panic;
use webcore::ffi;

/// Initializes the library.
///
/// Calling this is required for anything to work.
pub fn initialize() {
    static mut INITIALIZED: bool = false;
    unsafe {
        if INITIALIZED {
            return;
        }

        INITIALIZED = true;
    }

    js! { @(no_return)
        Module.STDWEB = {};
    };

    #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
    js! { @(no_return)
        Module.STDWEB.alloc = _malloc;
        Module.STDWEB.dyncall = Runtime.dynCall;
        Module.STDWEB.utf8_len = lengthBytesUTF8;
    };

    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
    js! { @(no_return)
        Module.STDWEB.alloc = Module.web_malloc;
        Module.STDWEB.dyncall = function( signature, ptr, args ) {
            return Module.web_table.get( ptr ).apply( null, args );
        };
        // This is based on code from Emscripten's preamble.js.
        Module.STDWEB.utf8_len = function utf8_len( str ) {
            let len = 0;
            for( let i = 0; i < str.length; ++i ) {
                // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
                // See http://unicode.org/faq/utf_bom.html#utf16-3
                let u = str.charCodeAt( i ); // possibly a lead surrogate
                if( u >= 0xD800 && u <= 0xDFFF ) {
                    u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt( ++i ) & 0x3FF);
                }

                if( u <= 0x7F ) {
                    ++len;
                } else if( u <= 0x7FF ) {
                    len += 2;
                } else if( u <= 0xFFFF ) {
                    len += 3;
                } else if( u <= 0x1FFFFF ) {
                    len += 4;
                } else if( u <= 0x3FFFFFF ) {
                    len += 5;
                } else {
                    len += 6;
                }
            }
            return len;
        };
    };

    js! { @(no_return)
        // This is based on code from Emscripten's preamble.js.
        Module.STDWEB.to_utf8 = function to_utf8( str, addr ) {
            for( var i = 0; i < str.length; ++i ) {
                // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
                // See http://unicode.org/faq/utf_bom.html#utf16-3
                // For UTF8 byte structure, see http://en.wikipedia.org/wiki/UTF-8#Description and https://www.ietf.org/rfc/rfc2279.txt and https://tools.ietf.org/html/rfc3629
                var u = str.charCodeAt( i ); // possibly a lead surrogate
                if( u >= 0xD800 && u <= 0xDFFF ) {
                    u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt( ++i ) & 0x3FF);
                }

                if( u <= 0x7F ) {
                    HEAPU8[ addr++ ] = u;
                } else if( u <= 0x7FF ) {
                    HEAPU8[ addr++ ] = 0xC0 | (u >> 6);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0xFFFF ) {
                    HEAPU8[ addr++ ] = 0xE0 | (u >> 12);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0x1FFFFF ) {
                    HEAPU8[ addr++ ] = 0xF0 | (u >> 18);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0x3FFFFFF ) {
                    HEAPU8[ addr++ ] = 0xF8 | (u >> 24);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 18) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else {
                    HEAPU8[ addr++ ] = 0xFC | (u >> 30);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 24) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 18) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                }
            }
        };
    };

    js! { @(no_return)
        Module.STDWEB.to_js = function to_js( address ) {
            var kind = HEAPU8[ address + 12 ];
            if( kind === 0 ) {
                return undefined;
            } else if( kind === 1 ) {
                return null;
            } else if( kind === 2 ) {
                return HEAP32[ address / 4 ];
            } else if( kind === 3 ) {
                return HEAPF64[ address / 8 ];
            } else if( kind === 4 ) {
                var pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                return Module.STDWEB.to_js_string( pointer, length );
            } else if( kind === 5 ) {
                return false;
            } else if( kind === 6 ) {
                return true;
            } else if( kind === 7 ) {
                var pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                var output = [];
                for( var i = 0; i < length; ++i ) {
                    output.push( Module.STDWEB.to_js( pointer + i * 16 ) );
                }
                return output;
            } else if( kind === 8 ) {
                var value_array_pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                var key_array_pointer = HEAPU32[ (address + 8) / 4 ];
                var output = {};
                for( var i = 0; i < length; ++i ) {
                    var key_pointer = HEAPU32[ (key_array_pointer + i * 8) / 4 ];
                    var key_length = HEAPU32[ (key_array_pointer + 4 + i * 8) / 4 ];
                    var key = Module.STDWEB.to_js_string( key_pointer, key_length );
                    var value = Module.STDWEB.to_js( value_array_pointer + i * 16 );
                    output[ key ] = value;
                }
                return output;
            } else if( kind === 9 || kind == 11 || kind == 12 ) {
                return Module.STDWEB.acquire_js_reference( HEAP32[ address / 4 ] );
            } else if( kind === 10 ) {
                var adapter_pointer = HEAPU32[ address / 4 ];
                var pointer = HEAPU32[ (address + 4) / 4 ];
                var deallocator_pointer = HEAPU32[ (address + 8) / 4 ];
                var output = function() {
                    var args = Module.STDWEB.alloc( 16 );
                    Module.STDWEB.serialize_array( args, arguments );
                    Module.STDWEB.dyncall( "vii", adapter_pointer, [pointer, args] );
                    var result = Module.STDWEB.tmp;
                    Module.STDWEB.tmp = null;

                    return result;
                };

                output.drop = function() {
                    output.drop = null;
                    Module.STDWEB.dyncall( "vi", deallocator_pointer, [pointer] );
                };
<commit_msg>`==` -> `===` in inline JS<commit_after>use std::panic;
use webcore::ffi;

/// Initializes the library.
///
/// Calling this is required for anything to work.
pub fn initialize() {
    static mut INITIALIZED: bool = false;
    unsafe {
        if INITIALIZED {
            return;
        }

        INITIALIZED = true;
    }

    js! { @(no_return)
        Module.STDWEB = {};
    };

    #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
    js! { @(no_return)
        Module.STDWEB.alloc = _malloc;
        Module.STDWEB.dyncall = Runtime.dynCall;
        Module.STDWEB.utf8_len = lengthBytesUTF8;
    };

    #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
    js! { @(no_return)
        Module.STDWEB.alloc = Module.web_malloc;
        Module.STDWEB.dyncall = function( signature, ptr, args ) {
            return Module.web_table.get( ptr ).apply( null, args );
        };
        // This is based on code from Emscripten's preamble.js.
        Module.STDWEB.utf8_len = function utf8_len( str ) {
            let len = 0;
            for( let i = 0; i < str.length; ++i ) {
                // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
                // See http://unicode.org/faq/utf_bom.html#utf16-3
                let u = str.charCodeAt( i ); // possibly a lead surrogate
                if( u >= 0xD800 && u <= 0xDFFF ) {
                    u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt( ++i ) & 0x3FF);
                }

                if( u <= 0x7F ) {
                    ++len;
                } else if( u <= 0x7FF ) {
                    len += 2;
                } else if( u <= 0xFFFF ) {
                    len += 3;
                } else if( u <= 0x1FFFFF ) {
                    len += 4;
                } else if( u <= 0x3FFFFFF ) {
                    len += 5;
                } else {
                    len += 6;
                }
            }
            return len;
        };
    };

    js! { @(no_return)
        // This is based on code from Emscripten's preamble.js.
        Module.STDWEB.to_utf8 = function to_utf8( str, addr ) {
            for( var i = 0; i < str.length; ++i ) {
                // Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code unit, not a Unicode code point of the character! So decode UTF16->UTF32->UTF8.
                // See http://unicode.org/faq/utf_bom.html#utf16-3
                // For UTF8 byte structure, see http://en.wikipedia.org/wiki/UTF-8#Description and https://www.ietf.org/rfc/rfc2279.txt and https://tools.ietf.org/html/rfc3629
                var u = str.charCodeAt( i ); // possibly a lead surrogate
                if( u >= 0xD800 && u <= 0xDFFF ) {
                    u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt( ++i ) & 0x3FF);
                }

                if( u <= 0x7F ) {
                    HEAPU8[ addr++ ] = u;
                } else if( u <= 0x7FF ) {
                    HEAPU8[ addr++ ] = 0xC0 | (u >> 6);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0xFFFF ) {
                    HEAPU8[ addr++ ] = 0xE0 | (u >> 12);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0x1FFFFF ) {
                    HEAPU8[ addr++ ] = 0xF0 | (u >> 18);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else if( u <= 0x3FFFFFF ) {
                    HEAPU8[ addr++ ] = 0xF8 | (u >> 24);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 18) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                } else {
                    HEAPU8[ addr++ ] = 0xFC | (u >> 30);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 24) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 18) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 12) & 63);
                    HEAPU8[ addr++ ] = 0x80 | ((u >> 6) & 63);
                    HEAPU8[ addr++ ] = 0x80 | (u & 63);
                }
            }
        };
    };

    js! { @(no_return)
        Module.STDWEB.to_js = function to_js( address ) {
            var kind = HEAPU8[ address + 12 ];
            if( kind === 0 ) {
                return undefined;
            } else if( kind === 1 ) {
                return null;
            } else if( kind === 2 ) {
                return HEAP32[ address / 4 ];
            } else if( kind === 3 ) {
                return HEAPF64[ address / 8 ];
            } else if( kind === 4 ) {
                var pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                return Module.STDWEB.to_js_string( pointer, length );
            } else if( kind === 5 ) {
                return false;
            } else if( kind === 6 ) {
                return true;
            } else if( kind === 7 ) {
                var pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                var output = [];
                for( var i = 0; i < length; ++i ) {
                    output.push( Module.STDWEB.to_js( pointer + i * 16 ) );
                }
                return output;
            } else if( kind === 8 ) {
                var value_array_pointer = HEAPU32[ address / 4 ];
                var length = HEAPU32[ (address + 4) / 4 ];
                var key_array_pointer = HEAPU32[ (address + 8) / 4 ];
                var output = {};
                for( var i = 0; i < length; ++i ) {
                    var key_pointer = HEAPU32[ (key_array_pointer + i * 8) / 4 ];
                    var key_length = HEAPU32[ (key_array_pointer + 4 + i * 8) / 4 ];
                    var key = Module.STDWEB.to_js_string( key_pointer, key_length );
                    var value = Module.STDWEB.to_js( value_array_pointer + i * 16 );
                    output[ key ] = value;
                }
                return output;
            } else if( kind === 9 || kind === 11 || kind === 12 ) {
                return Module.STDWEB.acquire_js_reference( HEAP32[ address / 4 ] );
            } else if( kind === 10 ) {
                var adapter_pointer = HEAPU32[ address / 4 ];
                var pointer = HEAPU32[ (address + 4) / 4 ];
                var deallocator_pointer = HEAPU32[ (address + 8) / 4 ];
                var output = function() {
                    var args = Module.STDWEB.alloc( 16 );
                    Module.STDWEB.serialize_array( args, arguments );
                    Module.STDWEB.dyncall( "vii", adapter_pointer, [pointer, args] );
                    var result = Module.STDWEB.tmp;
                    Module.STDWEB.tmp = null;

                    return result;
                };

                output.drop = function() {
                    output.drop = null;
                    Module.STDWEB.dyncall( "vi", deallocator_pointer, [pointer] );
                };

{'fraction_non_alphanumeric': 0.13714481342006163, 'fraction_numerical': 0.04313591235878124, 'mean_word_length': 1.2033489214059436, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 76, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5260903', 'n_tokens_mistral': 5174, 'n_tokens_neox': 4442, 'n_words': 1207}