<commit_before>var through = require('through2')
  , fs = require('fs')
  , debug = require('debug')('re-define:file:load')

module.exports = function(config) {
  return through.obj(function(module, enc, next){
    var self = this

    if(!module.path) {
      this.push(module)
      next()
      return
    }

    if (module.isNull()) {
      fs.exists(module.path, function(r) {
        if(r) {
          module.contents = fs.createReadStream(module.path)
          loadFromStream()
        } else {
          self.emit('not-exists', module)

          module.revert && module.revert()
          module.stopProcessing = true
          module.external = true

          debug('File does not exists, passing it further:', module, module.path)

          self.push(module)
          next()
        }
      })
    }

    if (module.isBuffer()) {
      this.push(module)
      next()
    }

    if (module.isStream()) loadFromStream()

    function loadFromStream() {
      module.pipe(through(function(content) {
        module.contents = content

        self.push(module)
        next()
      }, function(a) {
        module.contents = new Buffer('""')
        self.push(module)

        debug('File is empty:', module, module.path)
        next()
      }.bind(this)))
    }
  })
}
<commit_msg>resolved issue with loading larger files (like jquery), previous implementation was sooo wrong<commit_after>var through = require('through2')
  , fs = require('fs')
  , debug = require('debug')('re-define:file:load')

module.exports = function(config) {
  return through.obj(function(module, enc, next){
    var self = this

    if(!module.path) {
      this.push(module)
      next()
      return
    }

    if (module.isNull()) {
      fs.exists(module.path, function(r) {
        if(r) {
          module.contents = fs.createReadStream(module.path)
          loadFromStream()
        } else {
          self.emit('not-exists', module)

          module.revert && module.revert()
          module.stopProcessing = true
          module.external = true

          debug('File does not exists, passing it further:', module, module.path)

          self.push(module)
          next()
        }
      })
    }

    if (module.isBuffer()) {
      this.push(module)
      next()
    }

    if (module.isStream()) loadFromStream()

    function loadFromStream() {
      var buffers = []

      module.pipe(through(function(chunk, enc, next) {
          buffers.push(chunk)
          next()
      }, function(end) {
          module.contents = Buffer.concat(buffers)
          self.push(module)
          next()
          end()
      }.bind(this)))
    }
  })
}

{'fraction_non_alphanumeric': 0.13692946058091288, 'fraction_numerical': 0.0007544322897019992, 'mean_word_length': 2.051783659378596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '572224', 'n_tokens_mistral': 794, 'n_tokens_neox': 772, 'n_words': 157}