<commit_before>var path = require('path'),
    _ = require('lodash'),
    md5 = require('md5'),
    gulpIf = require('gulp-if'),
    File = require('vinyl'),
    del = require('del'),
    clc = require('cli-color'),
    eachInStream = require('../../util/eachInStream');

module.exports = storeAsyncPlugin;

var hashes = {},
    currentConfig = null,
    count = null,
    deleted = null;

/**
 * @ignore
 * Creates directory structure and stores modules and info files in a build path.
 * Modules are renamed regarding to MD5 hash sum of their contents.
 * Generates and stores `hashes.json` info file that matches short aliases of modules with hash-based names.
 * Used for asynchronous modules loading.
 * @alias "modules.storeAsync"
 * @returns {stream.Transform} Stream
 */
function storeAsyncPlugin (cfg) {
    currentConfig = cfg;
    count = 0;
    deleted = [];

    return gulpIf(isModule, eachInStream(processModule, createHashesFile));
}

function processModule (file, _, cb) {
    var hash = md5(file.contents),
        oldHash = hashes[file.moduleAlias];

    if (oldHash && oldHash != hash) {
        deleted.push(createPath(currentConfig.dest, oldHash));
    }

    hashes[file.moduleAlias] = hash;
    file.path = createPath(file.base, hash);
    count++;

    cb(null, file);
}

function isModule (file) {
    return file.isModule;
}

function createPath (base, hash) {
    return path.join(base, hash[0], hash[1], hash);
}

function createHashesFile (cb) {
    console.log(clc.green.bold('Stored ' + count + ' asynchronous modules.'));

    this.push(new File({
        path: path.join(process.cwd(), 'hashes.json'),
        contents: new Buffer(JSON.stringify(hashes, null, '\t'))
    }));

    del(deleted, { force: true }).then(function () { cb(); });
}<commit_msg>Getting rid of hashes.json in favor of using hashes for short aliases<commit_after>var path = require('path'),
    md5 = require('md5-jkmyers'),
    gulpIf = require('gulp-if'),
    eachInStream = require('../../util/eachInStream'),
    clc = require('cli-color');

module.exports = storeAsyncPlugin;

var count;

/**
 * @ignore
 * Renames modules regarding to MD5 hash sum of their short alias.
 * Used to store for asynchronous modules loading.
 * @alias "modules.storeAsync"
 * @returns {stream.Transform} Stream
 */
function storeAsyncPlugin () {
    count = 0;
    return gulpIf(isModule, eachInStream(processModule, printResults));
}

function processModule (file, enc, cb) {
    count++;
    file.path = createPath(file.base, md5(file.moduleAlias));
    cb(null, file);
}

function isModule (file) {
    return file.isModule;
}

function createPath (base, hash) {
    return path.join(base, hash[0], hash[1], hash);
}

function printResults (cb) {
    console.log(clc.green.bold('Stored ' + count + ' asynchronous module(s).'));
    cb();
}
{'fraction_non_alphanumeric': 0.14537755822159493, 'fraction_numerical': 0.004940014114326041, 'mean_word_length': 3.602272727272727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5880705', 'n_tokens_mistral': 890, 'n_tokens_neox': 874, 'n_words': 238}