<commit_before>var _ = require('lodash');
var fs = require('fs');
var path = require('path');
var async = require('async');
var BasePlugin = require('mixdown-app').Plugin;

module.exports = BasePlugin.extend({
  _namespace_default: 'config',
  init: function(options) {
    this._super(options);

    // this holds a pointer to the most recently loaded config.
    this._config = null;

    _.defaults(this._options, {
      paths: ['./config']
    });

    // ensure array
    if (!_.isArray(this._options.paths)) {
      this._options.paths = [this._options.paths];
    }
  },
  get: function(callback) {
    if (this._config) {
      callback(null, this._config);
    } else {
      throw new Error('External Config not initialized. Call setup().');
    }
  },
  crawl: function(callback) {

    var self = this;
    var full_paths = _.isArray(this._options.paths) ? this._options.paths : [this._options.paths];
    full_paths = _.map(full_paths, function(p) {
      return path.join(process.cwd(), p);
    });

    // get stats for all folders.
    var directory_stats = _.map(full_paths, function(file) {
      return function(cb) {

        var full_path = file;

        fs.exists(full_path, function(exists) {

          if (!exists) {
            cb(new Error('Directory does not exist.' + full_path));
            return;
          }
<commit_msg>mixdown 3 file system config module<commit_after>var _ = require('lodash');
var fs = require('fs');
var path = require('path');
var async = require('async');
var BasePlugin = require('mixdown-app').Plugin;

module.exports = BasePlugin.extend({
  _namespace_default: 'config',
  init: function(options) {
    this._super(options);

    // this holds a pointer to the most recently loaded config.
    this._config = null;
  },
  get: function(callback) {
    if (this._config) {
      callback(null, this._config);
    } else {
      throw new Error('External Config not initialized. Call setup().');
    }
  },
  crawl: function(callback) {

    // for not external config.
    if (!this._options.paths || this._options.paths.length === 0) {
      this._config = [];
      callback(null, this._config);
      return;
    }

    var self = this;
    var full_paths = _.isArray(this._options.paths) ? this._options.paths : [this._options.paths];
    full_paths = _.map(full_paths, function(p) {
      return path.join(process.cwd(), p);
    });

    // get stats for all folders.
    var directory_stats = _.map(full_paths, function(file) {
      return function(cb) {

        var full_path = file;

        fs.exists(full_path, function(exists) {

          if (!exists) {
            cb(new Error('Directory does not exist.' + full_path));
            return;
          }

{'fraction_non_alphanumeric': 0.14222873900293256, 'fraction_numerical': 0.0007331378299120235, 'mean_word_length': 2.717983651226158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2379753', 'n_tokens_mistral': 829, 'n_tokens_neox': 811, 'n_words': 204}