<commit_before>/// <reference path="emitter/util.ts"/>

namespace ks {
    declare var require:any;
    
    function getLzma() {
        if (U.isNodeJS) return require("lzma");
        else return (<any>window).LZMA;
    }
    
    export function lzmaDecompressAsync(buf: Uint8Array): Promise<string> { // string
        return new Promise<string>((resolve, reject) => {
            try {
                getLzma().decompress(buf, (res: string, error: any) => {
                    resolve(error ? undefined : res);
                })
            }
            catch (e) {
                resolve(undefined);
            }
        })
    }

    export function lzmaCompressAsync(text: string): Promise<Uint8Array> {
        return new Promise<Uint8Array>((resolve, reject) => {
            try {
                getLzma().compress(text, 7, (res: any, error: any) => {
                    resolve(error ? undefined : new Uint8Array(res));
                })
            }
            catch (e) {
                resolve(undefined);
            }
        })
    }
}

namespace ks.cpp {
    import U = ts.ks.Util;
    import Y = ts.ks;<commit_msg>Fail early if lzma module missing<commit_after>/// <reference path="emitter/util.ts"/>

namespace ks {
    declare var require:any;
    
    function getLzma() {
        if (U.isNodeJS) return require("lzma");
        else return (<any>window).LZMA;
    }
    
    export function lzmaDecompressAsync(buf: Uint8Array): Promise<string> { // string
        let lzma = getLzma()
        return new Promise<string>((resolve, reject) => {
            try {
                lzma.decompress(buf, (res: string, error: any) => {
                    resolve(error ? undefined : res);
                })
            }
            catch (e) {
                resolve(undefined);
            }
        })
    }

    export function lzmaCompressAsync(text: string): Promise<Uint8Array> {
        let lzma = getLzma()
        return new Promise<Uint8Array>((resolve, reject) => {
            try {
                lzma.compress(text, 7, (res: any, error: any) => {
                    resolve(error ? undefined : new Uint8Array(res));
                })
            }
            catch (e) {
                resolve(undefined);
            }
        })
    }
}

namespace ks.cpp {
    import U = ts.ks.Util;
    import Y = ts.ks;
{'fraction_non_alphanumeric': 0.12457627118644068, 'fraction_numerical': 0.00423728813559322, 'mean_word_length': 1.538709677419355, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2303645', 'n_tokens_mistral': 704, 'n_tokens_neox': 678, 'n_words': 150}