<commit_before>"use strict";

var _fs = require('fs'),
    EventEmitter = require('events').EventEmitter,
    createFilterStatus = require('./filterStatus'),
    async = require('async');

function _nodeStat (path, stat) {
  var filterStatus = createFilterStatus();
  this._filters.forEach(function(filter) {
    filter.call(filterStatus.facade, path, stat);
  });
  if (!filterStatus.droppedNode()) {
    this.emit('path', path, stat);
  }
  if (stat.isFile()) {
    this.emit('end');
  } else if (stat.isDirectory() && !filterStatus.brokenNode()) {
    _fs.readdir(path, function (err, children) {
      if (err) {
        return this.emit('error', err);
      }
      _readNode.call(this, path, children);
    }.bind(this));
  }
}

function _readNode(path, children) {
  children = children.map(function (c) {
    var treeverse = new Treeverse();
    treeverse._filters = this._filters;
    treeverse.on('path', this.emit.bind(this, 'path'));
    return function (cb) {
      treeverse.on('error', cb.bind(this));
      treeverse.on('end', cb.bind(this, undefined));<commit_msg>fixes the break on non-empty dirs<commit_after>"use strict";

var _fs = require('fs'),
    EventEmitter = require('events').EventEmitter,
    createFilterStatus = require('./filterStatus'),
    async = require('async');

function _nodeStat (path, stat) {
  var filterStatus = createFilterStatus();
  this._filters.forEach(function(filter) {
    filter.call(filterStatus.facade, path, stat);
  });
  if (!filterStatus.droppedNode()) {
    this.emit('path', path, stat);
  }
  if (stat.isFile()) {
    this.emit('end');
  } else if (stat.isDirectory()) {
    if (filterStatus.brokenNode()) {
      this.emit('end');
    } else {
      _fs.readdir(path, function (err, children) {
        if (err) {
          return this.emit('error', err);
        }
        _readNode.call(this, path, children);
      }.bind(this));
    }
  }
}

function _readNode(path, children) {
  children = children.map(function (c) {
    var treeverse = new Treeverse();
    treeverse._filters = this._filters;
    treeverse.on('path', this.emit.bind(this, 'path'));
    return function (cb) {
      treeverse.on('error', cb.bind(this));
      treeverse.on('end', cb.bind(this, undefined));
{'fraction_non_alphanumeric': 0.17023172905525846, 'fraction_numerical': 0.0, 'mean_word_length': 3.49, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1303538', 'n_tokens_mistral': 715, 'n_tokens_neox': 714, 'n_words': 147}