<commit_before>function qbp(opts) {
    var options = {
        threads: 1,
        progressInterval: 10000,
        progress: noop,
        empty: noop
    };

    var _this = this;

    _this.status = 'waiting';

    var queue = [];
    var running = false;
    var itemCount = 0;
    var completeCount = 0;
    var threadCount = 0;
    var lastCompleteCount = 0;

    for (var key in options) {
        if (opts[key] === undefined) {
            opts[key] = options[key];
        }
    }

    if (opts.items) {
        add(opts.items);
    }

    function add(itemOrArray) {
        if (itemOrArray instanceof Array) {
            itemCount += itemOrArray.length;
            queue = queue.concat(queue, itemOrArray);
        }
        else {
            itemCount++;
            queue.push(itemOrArray);
        }

        resume();
    }

    function empty() {
        queue.length = 0;
    }

    function resume() {
        if (!running) {
            running = true;
            _this.status = 'running';
            setupThreads(true);
            if (queue.length > 0 || completeCount < itemCount) {
                progress();
            }
        }
    }

    function pause() {
        running = false;
        _this.status = 'paused';
    }

    function progress(once) {
        if (!running && !once) return;

        var perc;
        if (itemCount > 0) perc = completeCount / itemCount;
        else perc = 0;

        var newItemsCompleted = completeCount - lastCompleteCount;
        var timeDiff = 1000 / opts.progressInterval;
        var itemsPerSecond = Math.round(newItemsCompleted * timeDiff);<commit_msg>Improving the pause and resume functionality<commit_after>function qbp(opts) {
    var options = {
        threads: 1,
        progressInterval: 10000,
        progress: noop,
        empty: noop
    };

    var _this = this;

    _this.status = 'waiting';

    var queue = [];
    var running = false;
    var itemCount = 0;
    var completeCount = 0;
    var threadCount = 0;
    var lastCompleteCount = 0;

    for (var key in options) {
        if (opts[key] === undefined) {
            opts[key] = options[key];
        }
    }

    if (opts.items) {
        add(opts.items);
    }

    function add(itemOrArray) {
        if (itemOrArray instanceof Array) {
            itemCount += itemOrArray.length;
            queue = queue.concat(queue, itemOrArray);
        }
        else {
            itemCount++;
            queue.push(itemOrArray);
        }

        resume(true);
    }

    function empty() {
        queue.length = 0;
    }

    function resume(newItem) {
        if (!running && (!newItem || _this.status !== 'paused')) {
            running = true;
            _this.status = 'running';
            setupThreads(true);
            if (queue.length > 0 || completeCount < itemCount) {
                progress();
            }
        }
    }

    function pause() {
        running = false;
        _this.status = 'paused';
    }

    function progress(once) {
        if (!running && !once) return;

        var perc;
        if (itemCount > 0) perc = completeCount / itemCount;
        else perc = 0;

        var newItemsCompleted = completeCount - lastCompleteCount;
        var timeDiff = 1000 / opts.progressInterval;
        var itemsPerSecond = Math.round(newItemsCompleted * timeDiff);
{'fraction_non_alphanumeric': 0.10951954640405849, 'fraction_numerical': 0.010743061772605193, 'mean_word_length': 1.6666666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2660919', 'n_tokens_mistral': 1009, 'n_tokens_neox': 936, 'n_words': 239}