<commit_before>var fs = require("fs"),
    utils = require("utils"),
    filePath = require("file_path");


var fileUtils = module.exports;


fileUtils.diveDefaultOpts = {
    all: false,
    recursive: true,
    files: true,
    directories: false
};

fileUtils.dive = function dive(dir, opts, action, complete) {
    var todo = 1;

    if (utils.isFunction(opts)) {
        if (utils.isFunction(action)) {
            complete = action;
        } else {
            complete = utils.noop;
        }

        action = opts;
        opts = {};
    } else if (complete == null) {
        complete = utils.noop;
    }
    if (!utils.isString(dir)) dir = process.cwd();

    utils.mixin(opts, fileUtils.diveDefaultOpts);

    (function doDive(dir) {

        fs.readdir(dir, function(err, files) {
            if (err) {
                action(err);
                return;
            }

            todo--;

            utils.each(files, function(file) {
                if (opts.all || file[0] !== ".") {
                    var fullPath = filePath.resolve(dir, file),
                        stat;

                    todo++;

                    try {
                        stat = fs.statSync(fullPath);
                    } catch (err) {
                        todo--;
                        return action(err) !== false;
                    }

                    if (stat) {
                        if (stat.isDirectory()) {
                            if (opts.directories) return action(null, fullPath) !== false;
                            if (opts.recursive) doDive(fullPath);
                        } else {
                            if (opts.files) return action(null, fullPath) !== false;
                        }
                    }

                    return true;
                }

                return true;
            });

            if (!todo) complete();
        });
    }(dir));
};

fileUtils.diveSync = function dive(dir, opts, action) {
    var todo = 1;

    if (utils.isFunction(opts)) {
        action = opts;
        opts = {};
    }
    if (!utils.isString(dir)) dir = process.cwd();

    utils.mixin(opts, fileUtils.diveDefaultOpts);

    (function doDive(dir) {
        try {
            files = fs.readdirSync(dir);
        } catch (err) {
            action(err);
            return;
        }
        todo--;

        utils.each(files, function(file) {<commit_msg>updates<commit_after>var fs = require("fs"),
    utils = require("utils"),
    filePath = require("file_path");


var fileUtils = module.exports;


fileUtils.diveDefaults = {
    all: false,
    recursive: true,
    files: true,
    directories: false
};

fileUtils.dive = function dive(dir, opts, action, complete) {
    var todo = 1;

    if (utils.isFunction(opts)) {
        if (utils.isFunction(action)) {
            complete = action;
        } else {
            complete = utils.noop;
        }

        action = opts;
        opts = {};
    } else if (complete == null) {
        complete = utils.noop;
    }
    if (!utils.isString(dir)) dir = process.cwd();

    utils.mixin(opts, fileUtils.diveDefaults);

    (function doDive(dir) {

        fs.readdir(dir, function(err, files) {
            if (err) {
                action(err);
                return;
            }

            todo--;

            utils.each(files, function(file) {
                if (opts.all || file[0] !== ".") {
                    var fullPath = filePath.resolve(dir, file),
                        stat;

                    todo++;

                    try {
                        stat = fs.statSync(fullPath);
                    } catch (err) {
                        todo--;
                        return action(err) !== false;
                    }

                    if (stat) {
                        if (stat.isDirectory()) {
                            if (opts.directories) return action(null, fullPath) !== false;
                            if (opts.recursive) doDive(fullPath);
                        } else {
                            if (opts.files) return action(null, fullPath) !== false;
                        }
                    }

                    return true;
                }

                return true;
            });

            if (!todo) complete();
        });
    }(dir));
};

fileUtils.diveSync = function dive(dir, opts, action) {
    var todo = 1;

    if (utils.isFunction(opts)) {
        action = opts;
        opts = {};
    }
    if (!utils.isString(dir)) dir = process.cwd();

    utils.mixin(opts, fileUtils.diveDefaults);

    (function doDive(dir) {
        try {
            files = fs.readdirSync(dir);
        } catch (err) {
            action(err);
            return;
        }
        todo--;

        utils.each(files, function(file) {
{'fraction_non_alphanumeric': 0.12419522326064382, 'fraction_numerical': 0.0012461059190031153, 'mean_word_length': 1.2041189931350114, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7338563', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1293, 'n_words': 282}