<commit_before>'use strict';

// Based on gulp-size by @sindresorhus

var chalk = require('chalk');
var gzipSize = require('gzip-size');
var mixIn = require('mout/object/mixIn');
var prettyBytes = require('pretty-bytes');

module.exports = function (options) {
    options = mixIn({
        showFiles: true,
        gzip: true
    }, options || {});

    return function (tree) {
        var result = [];
        var totalSize = 0;
        var remaining = tree.nodes.length;

        tree.nodes.forEach(function (node, i) {
            result.push(null);

            function finish(err, size) {
                result[i] = { name: node.name, size: size };

                totalSize += size;

                if (--remaining === 0) {
                    result.forEach(function (file) {
                        if (options.showFiles === true && file.size > 0) {
                            log(options.title, chalk.blue(file.name), file.size, options.gzip);
                        }
                    });
                    if (result.length !== 1 || !options.showFiles || totalSize === 0) {
                        log(options.title, chalk.green('total'), totalSize, options.gzip);
                    }
                }
            }

            if (options.gzip) {
                gzipSize(node.data, finish);
            } else {
                finish(null, node.data.length);
            }
        });

        return tree;
    };
};

function log(title, what, size, gzip) {
    title = title ? ('\'' + chalk.cyan(title) + '\' ') : '';
    console.log(title + what + ' ' + chalk.magenta(prettyBytes(size)) +
        (gzip ? chalk.gray(' (gzipped)') : ''));
}<commit_msg>Improve the stats output<commit_after>'use strict';

// Based on gulp-size by @sindresorhus

var chalk = require('chalk');
var gzipSize = require('gzip-size');
var mixIn = require('mout/object/mixIn');
var prettyBytes = require('pretty-bytes');

module.exports = function (options) {
    options = mixIn({
        showFiles: true
    }, options || {});

    return function (tree) {
        var result = [];
        var rawTotalSize = 0;
        var gzTotalSize = 0;
        var remaining = tree.nodes.length;

        tree.nodes.forEach(function (node, i) {
            result.push(null);

            gzipSize(node.data, function (err, gzSize) {
                result[i] = { name: node.name, rawSize: node.data.length, gzSize: gzSize };

                rawTotalSize += node.data.length;
                gzTotalSize += gzSize;

                if (--remaining === 0) {
                    result.forEach(function (file) {
                        if (options.showFiles === true && file.rawSize > 0) {
                            log(options.title, chalk.blue(file.name), file.rawSize, file.gzSize);
                        }
                    });
                    if (result.length !== 1 || !options.showFiles || rawTotalSize === 0) {
                        log(options.title, chalk.green('total'), rawTotalSize, gzTotalSize);
                    }
                }
            });
        });

        return tree;
    };
};

function log(title, what, rawSize, gzSize) {
    title = title ? ('\'' + chalk.cyan(title) + '\' ') : '';
    console.log(title + what + ' ' + chalk.gray(prettyBytes(rawSize)) +
        chalk.magenta(' (' + prettyBytes(gzSize) + ' gzipped)'));
}
{'fraction_non_alphanumeric': 0.1488855869242199, 'fraction_numerical': 0.0032689450222882616, 'mean_word_length': 1.6524822695035462, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7099400', 'n_tokens_mistral': 1015, 'n_tokens_neox': 949, 'n_words': 204}