<commit_before>var free = require("free-memory"),
    bytesized = require("bytesized"),
    createMonitor = require("./monitor");

/**
 * Create free memory monitor.
 * @param {object} opts
 * @param {string|number} opts.threshold
 * @param {number} [opts.headroom]
 * @param {Console} [opts.console]
 */
function mem(opts) {
    var threshold,
        headroom = 5,
        monitor,
        worker;

    if (!opts.threshold) throw new TypeError("threshold required");
    if (opts.headroom) headroom = parseInt(opts.headroom);
    threshold = bytesized(opts.threshold);

    /**
     * Take a sample.
     */
    function sample() {
        monitor.debug("taking a memory sample");

        free(function(err, info) {
            var delay;

            if (err) {
                monitor.emit("error", err);
            } else if (failed(info.mem)) {
                monitor.fail(info.mem);
            } else if (cleared(info.mem)) {
                monitor.clear();
            }

            delay = monitor.state === "failed" ? 15000 : 1500;
            worker = setTimeout(sample, delay);
        });
    }

    /**
     * Test sample against threshold.
     * @param {object} sample
     * @returns {boolean}
     */
    function failed(sample) {
        // NOTE: use .cached value; seems like free-memory module reports
        // .free+.buffers+.cached simply as .cached; .usable is junk
        return threshold < 1
            ? sample.cached/sample.total < threshold
            : sample.cached < threshold;
    }

    /**
     * Test sample against clear threshold.
     * @param {object} sample
     * @returns {boolean}
     */
    function cleared(sample) {
        var factor = 1 + headroom / 100;

        return threshold < 1
            ? sample.cached/sample.total * factor > threshold
            : sample.cached * factor > threshold;
    }
    
    /**
     * Cancel monitor.
     */
    function cancel() {
        worker && clearTimeout(worker);
        worker = null;
    }

    monitor = createMonitor(cancel, opts.console);
    worker = setTimeout(sample, 0);

    return monitor;
}

module.exports = mem;<commit_msg>--fix-free-memory option<commit_after>var free = require("free-memory"),
    bytesized = require("bytesized"),
    createMonitor = require("./monitor");

/**
 * Create free memory monitor.
 * @param {object} opts
 * @param {string|number} opts.threshold
 * @param {number} [opts.headroom]
 * @param {Console} [opts.console]
 */
function mem(opts) {
    var threshold,
        headroom = 5,
        monitor,
        worker;

    if (!opts.threshold) throw new TypeError("threshold required");
    if (opts.headroom) headroom = parseInt(opts.headroom);
    threshold = bytesized(opts.threshold);

    /**
     * Take a sample.
     */
    function sample() {
        monitor.debug("taking a memory sample");

        free(function(err, info) {
            var delay;

            if (err) {
                monitor.emit("error", err);
            } else if (failed(info.mem)) {
                monitor.fail(info.mem);
            } else if (cleared(info.mem)) {
                monitor.clear();
            }

            delay = monitor.state === "failed" ? 15000 : 1500;
            worker = setTimeout(sample, delay);
        });
    }

    /**
     * Test sample against threshold.
     * @param {object} sample
     * @returns {boolean}
     */
    function failed(sample) {
        var fix, key;

        // on some platforms, free-memory seems to report memory incorrectly
        // use --fix-free-memory to work around this
        fix = Boolean(~process.argv.indexOf("--fix-free-memory"));
        key = fix ? "cached" : "usable";

        return threshold < 1
            ? sample[key]/sample.total < threshold
            : sample[key] < threshold;
    }

    /**
     * Test sample against clear threshold.
     * @param {object} sample
     * @returns {boolean}
     */
    function cleared(sample) {
        var factor = 1 + headroom / 100;

        return threshold < 1
            ? sample.cached/sample.total * factor > threshold
            : sample.cached * factor > threshold;
    }

    /**
     * Cancel monitor.
     */
    function cancel() {
        worker && clearTimeout(worker);
        worker = null;
    }

    monitor = createMonitor(cancel, opts.console);
    worker = setTimeout(sample, 0);

    return monitor;
}

module.exports = mem;
{'fraction_non_alphanumeric': 0.1298142274580879, 'fraction_numerical': 0.007702763932940644, 'mean_word_length': 1.9277188328912467, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7275847', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1197, 'n_words': 316}