<commit_before>'use strict';

const rimraf = require('rimraf'),
    path = require('path'),
    async = require('async'),
    lib = require('../lib'),
    logger = lib.helpers.logger,
    config = require('../config');

module.exports = (name, callback) => {
    let blobPath = config.paths.blobs + "*",
        tokenPath = config.paths.tokens + "*";
    if(!!name) {
        blobPath = path.join(__dirname, "../", lib.helpers.dumpName(name) );
        tokenPath = path.join(__dirname, "../", lib.helpers.tokenDumpName(name) );
    }
    return async.parallel({
        flushBlob: (cb) => {
            rimraf(blobPath, function() {
                cb(null, "Flushed blob(s)");
            });
        },
        flushTokens: (cb) => {
            rimraf(tokenPath, function() {
                cb(null, "Flushed token(s)");
            });
        }
    }, function(err, data) {
        if(err) {
            logger.error(err);
        } else {
            logger.info(data);
        }
        callback();
    });

    //too lazy to write different messages for individual files -_-
}<commit_msg>Start flushing metas as well<commit_after>'use strict';

const rimraf = require('rimraf'),
    path = require('path'),
    async = require('async'),
    lib = require('../lib'),
    logger = lib.helpers.logger,
    config = require('../config');

module.exports = (name, callback) => {
    let blobPath = config.paths.blobs + "*",
        tokenPath = config.paths.tokens + "*",
        metaPath = config.paths.metas + "*";
    if(!!name) {
        blobPath = path.join(__dirname, "../", lib.helpers.dumpName(name) );
        tokenPath = path.join(__dirname, "../", lib.helpers.tokenDumpName(name) );
        metaPath = path.join(__dirname, "../", lib.helpers.metaName(name) );
    }
    return async.parallel({
        flushBlob: (cb) => {
            rimraf(blobPath, function() {
                cb(null, "Flushed blob(s)");
            });
        },
        flushTokens: (cb) => {
            rimraf(tokenPath, function() {
                cb(null, "Flushed token(s)");
            });
        },
        flushMetas: (cb) => {
            rimraf(metaPath, function() {
                cb(null, "Flushed Meta(s)");
            });
        }
    }, function(err, data) {
        if(err) {
            logger.error(err);
        } else {
            logger.info(data);
        }
        callback();
    });

    //too lazy to write different messages for individual files -_-
}
{'fraction_non_alphanumeric': 0.17212121212121212, 'fraction_numerical': 0.0, 'mean_word_length': 1.8657407407407407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1913408', 'n_tokens_mistral': 726, 'n_tokens_neox': 731, 'n_words': 140}