<commit_before>var uglify = require("uglify-js"),
    fs = require("fs"),
    p = require("path"),
    files = [
        "lzma_worker.js",
        "lzma.js",
    ];

function filesize(path, cb)
{
    return fs.statSync(path).size;
}


function sort_obj(obj)
{
    return Object.keys(obj).sort(function sorter(a, b)
    {
        return obj[b] - obj[a];
    });
}

function calculate_minify_value(props)
{
    Object.keys(props).forEach(function oneach(prop])
    {
        props[prop]] = prop].length * props[prop]];
    });
}

function base54_64(arr)
{
    var chars = "_$abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
        i,
        len = arr.length,
        new_names = [],
        str,
        pos,
        base;
    
    for (i = 0; i < len; i += 1) {
        pos = i;
        str = "";
        base = 54;
        do {
            ///NOTE: This is not perfect. It skips the first char (_) on the last place when more than one letter.
            str += chars[pos % base];
            pos = Math.floor(pos / base);
            /// After the first character, we can use numbers.
            base = 64;
        } while (pos > 0);
        new_names[i] = str;
    }
    
    return new_names;
}

function minify_properties(code)
{
    var props = {},
        ignore = ["LZMA_WORKER", "callback_num", "on_progress", "on_finish"],
        sorted_props,
        new_names,
        prop_regex = /\.([_$a-zA-Z]*[_$][_$a-zA-Z0-9]*|kFixHashSize|kNumHashDirectBytes|kMinMatchCheck|outBytesProcessed|decoder|encoder|nowPos64|outSize|alive|processedInSize|finished|inBytesProcessed|processedInSize|[Ss]tate|prevByte|Prev1IsChar|Prev2|BackPrev|PosPrev|Models|backRes|properties|processedOutSize|tempPrices|backRes|repLens|Price|Backs[0123]|NumBitLevels|Range|Stream|explicitLength|count|pos|buf|chunker|rep[0-3s]|Code|Low|data|mode|output)/g;
    
    /// We want to replace prop]eters that have an underscore or a dollar sign.
    code.replace(prop_regex, function calc(prop])
    {
        prop] = prop].substr(1);
        if (ignore.indexOf(prop]) === -1) {
            if (!props[prop]]) {
                props[prop]] = 1;
            } else {
                props[prop]] += 1;
            }
        }
    });
    
    //console.log(props);
    //console.log(Object.keys(props).length);
    calculate_minify_value(props);
    sorted_props = sort_obj(props);
    
    //console.log(props);
    //console.log(sorted_props);
    /*
    sorted_props.forEach(function (prop])
    {
        console.log(prop], props[prop]])
    });
    */
    new_names = base54_64(sorted_props);
    /*
    sorted_props.forEach(function (prop], i)
    {
        console.log(prop], props[prop]], new_names[i])
    });
    */
    
    code = code.replace(prop_regex, function calc(prop])
    {
        var index,
            partial_prop] = prop].substr(1);
        index = sorted_props.indexOf(partial_prop]);
        //console.log(prop], index);
        if (index > -1) {
            return "." + new_names[index];
        }
        return prop];
    });
    
    return code;
}


files.forEach(function oneach(file)
{
    var full_path = p.join(__dirname, "src", file),
        min_path = p.join(__dirname, "src", p.basename(file, p.extname(file)) + "-min" + p.extname(file)),
        result,
        orig_size = filesize(full_path),<commit_msg>Fixed find and replace goof.<commit_after>var uglify = require("uglify-js"),
    fs = require("fs"),
    p = require("path"),
    files = [
        "lzma_worker.js",
        "lzma.js",
    ];

function filesize(path, cb)
{
    return fs.statSync(path).size;
}


function sort_obj(obj)
{
    return Object.keys(obj).sort(function sorter(a, b)
    {
        return obj[b] - obj[a];
    });
}

function calculate_minify_value(props)
{
    Object.keys(props).forEach(function oneach(prop)
    {
        props[prop] = prop.length * props[prop];
    });
}

function base54_64(arr)
{
    var chars = "_$abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789",
        i,
        len = arr.length,
        new_names = [],
        str,
        pos,
        base;
    
    for (i = 0; i < len; i += 1) {
        pos = i;
        str = "";
        base = 54;
        do {
            ///NOTE: This is not perfect. It skips the first char (_) on the last place when more than one letter.
            str += chars[pos % base];
            pos = Math.floor(pos / base);
            /// After the first character, we can use numbers.
            base = 64;
        } while (pos > 0);
        new_names[i] = str;
    }
    
    return new_names;
}

function minify_properties(code)
{
    var props = {},
        ignore = ["LZMA_WORKER", "callback_num", "on_progress", "on_finish"],
        sorted_props,
        new_names,
        prop_regex = /\.([_$a-zA-Z]*[_$][_$a-zA-Z0-9]*|kFixHashSize|kNumHashDirectBytes|kMinMatchCheck|outBytesProcessed|decoder|encoder|nowPos64|outSize|alive|processedInSize|finished|inBytesProcessed|processedInSize|[Ss]tate|prevByte|Prev1IsChar|Prev2|BackPrev|PosPrev|Models|backRes|properties|processedOutSize|tempPrices|backRes|repLens|Price|Backs[0123]|NumBitLevels|Range|Stream|explicitLength|count|pos|buf|chunker|rep[0-3s]|Code|Low|data|mode|output)/g;
    
    /// We want to replace propeters that have an underscore or a dollar sign.
    code.replace(prop_regex, function calc(prop)
    {
        prop = prop.substr(1);
        if (ignore.indexOf(prop) === -1) {
            if (!props[prop]) {
                props[prop] = 1;
            } else {
                props[prop] += 1;
            }
        }
    });
    
    //console.log(props);
    //console.log(Object.keys(props).length);
    calculate_minify_value(props);
    sorted_props = sort_obj(props);
    
    //console.log(props);
    //console.log(sorted_props);
    /*
    sorted_props.forEach(function (prop)
    {
        console.log(prop, props[prop])
    });
    */
    new_names = base54_64(sorted_props);
    /*
    sorted_props.forEach(function (prop, i)
    {
        console.log(prop, props[prop], new_names[i])
    });
    */
    
    code = code.replace(prop_regex, function calc(prop)
    {
        var index,
            partial_prop = prop.substr(1);
        index = sorted_props.indexOf(partial_prop);
        //console.log(prop, index);
        if (index > -1) {
            return "." + new_names[index];
        }
        return prop;
    });
    
    return code;
}


files.forEach(function oneach(file)
{
    var full_path = p.join(__dirname, "src", file),
        min_path = p.join(__dirname, "src", p.basename(file, p.extname(file)) + "-min" + p.extname(file)),
        result,
        orig_size = filesize(full_path),
{'fraction_non_alphanumeric': 0.14611119436535291, 'fraction_numerical': 0.01288775663120036, 'mean_word_length': 2.3336663336663337, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5669366', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2260, 'n_words': 395}