<commit_before>var Mine = function() {

  function Mine() {
    this.dict = {};
    this.order = [];
  }

  var search = Mine.prototype;

  search.mineFile = function(files, callback) {

    if(typeof files === 'string') {
      files = [files];
    }

    this._mineFile(files, callback);

  };

  search._mineFile = function(files, callback, i, bigData) {

    if(!i) {
      i = 0; 
    }

    if(bigData) {
      bigData += bigData;
    }

    fs.readFile(files[i], 'utf8', function(err, data) {
    
      if(err) {
        console.log(err);
      }

      if(i < files.length - 1) {
        return mine(files, callback, ++i, data);
      }

      bigData += data;

      return callback(bigData);

    });

  };

  return Mine;

})();

exports = module.exports = new Mine();<commit_msg>Mine one or multiple files<commit_after>var fs = require('fs');

var Mine = (function() {

  function Mine() {
    this.dict = {};
    this.order = [];
  }

  var search = Mine.prototype;

  search.parseText = function(text, options, callback) {
    
    text = text
      .replace(/(\r\n|\n|\r)/gm,' ') // Remove line break 
      .replace(/\s+/g,' ') // Remove extra whitespace
      .replace(/\W/g, ' '); // Remove non alphanumerical

    var words = text.split(' ');
    var self = this;

    words.forEach(function(word) {

      word = word.toLowerCase();

      self.dict[word] = self.dict[word] 
      ? ++self.dict[word]
      : 1;

    });

    delete this.dict['']; // Remove empty space

    callback = arguments.length === 2
      ? options
      : callback;

    return callback(this.dict);

  };

  search.parseFile = function(files, options, callback) {

    if(typeof files === 'string') {
      files = [files];
    }

    callback = arguments.length === 2
      ? options
      : callback;

    this._mineFile(files, callback);

  };

  search._mineFile = function(files, callback, i, bigData) {

    if(!i) {
      i = 0; 
    }

    if(bigData) {
      bigData += bigData;
    }

    var self = this;

    fs.readFile(files[i], 'utf8', function(err, data) {
    
      if(err) {
        console.log(err);
      }

      if(i < files.length - 1) {
        return self._mineFile(files, callback, ++i, data);
      }

      bigData += data;

      return self.parseText(bigData, callback);

    });

  };

  return Mine;

})();

exports = module.exports = new Mine();
{'fraction_non_alphanumeric': 0.16407236011779555, 'fraction_numerical': 0.0037862852334875894, 'mean_word_length': 2.088311688311688, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7074737', 'n_tokens_mistral': 824, 'n_tokens_neox': 768, 'n_words': 169}