<commit_before>var fs = require('fs');
var util = require('util');
var events = require('events');
var path = require('path');
var Bagpipe = require('bagpipe');

function Walker(option) {
  events.EventEmitter.call(this);
  var that = this;
  this.on('walk', function (_path, isDirectory) {
    if (isDirectory) {
      that.emit('folder', _path);
    } else {
      that.emit('file', _path);
    }
    if (that.counter < 0) {
      // 让业务逻辑先执行
      process.nextTick(function () {
        that.emit('end');
      });
    }
  });
  this.counter = 0;
  this.bagpipe = new Bagpipe(500);
  //默认的过滤器为空
  this.filterDir = [];
  if(option.filterDir){
    this.filterDir = option.filterDir
  }
}

util.inherits(Walker, events.EventEmitter);

/**
 * @param {String} _path 起始路径
 */
Walker.prototype.getList = function (_path) {
  var that = this;
  fs.lstat(_path, function (err, stat) {
    if (err) {
      that.emit('error', err);
    }
    var isDirectory = stat.isDirectory();
    if (isDirectory) {
      var pathArr = _path.split(path.sep);
      var dirName = pathArr[pathArr.length-1];
      //应用过滤器
      if(~that.filterDir.indexOf(dirName)){
        that.counter = that.counter - 1;
      }else{
        // 防止文件描述符打开过多
        that.bagpipe.push(fs.readdir, [_path], function (err, files) {
          if (err) {
              that.emit('error', err);
          }
          that.counter = that.counter + files.length - 1;
          files.forEach(function (filename) {
              that.getList(path.join(_path, filename));
          });
          that.emit('walk', path.resolve(_path), isDirectory);
        });
      }

    } else {
      that.counter = that.counter - 1;
      that.emit('walk', path.resolve(_path), isDirectory);
    }
  });
};

// 遍历方法
Walker.prototype.walk = function (source, callback) {
  if (typeof callback === 'function') {
    this.on('walk', callback);
  }
  this.getList(source);
};

module.exports = Walker;<commit_msg>修复代码风格的问题<commit_after>var fs = require('fs');
var util = require('util');
var events = require('events');
var path = require('path');
var Bagpipe = require('bagpipe');

function Walker(option) {
  events.EventEmitter.call(this);
  var that = this;
  this.on('walk', function (_path, isDirectory) {
    if (isDirectory) {
      that.emit('folder', _path);
    } else {
      that.emit('file', _path);
    }
    if (that.counter < 0) {
      // 让业务逻辑先执行
      process.nextTick(function () {
        that.emit('end');
      });
    }
  });
  this.counter = 0;
  this.bagpipe = new Bagpipe(500);
  this.filterDir = [];
  if (option && option.filterDir) {
    this.filterDir = option.filterDir;
  }
}

util.inherits(Walker, events.EventEmitter);

/**
 * @param {String} _path 起始路径
 */
Walker.prototype.getList = function (_path) {
  var that = this;
  fs.lstat(_path, function (err, stat) {
    if (err) {
      that.emit('error', err);
    }
    var isDirectory = stat.isDirectory();
    if (isDirectory) {
      var dirName = path.basename(_path);
      //应用过滤器
      if ( that.filterDir.indexOf(dirName) !== -1 ) {
        that.counter = that.counter - 1;
      } else {
        // 防止文件描述符打开过多
        that.bagpipe.push(fs.readdir, [_path], function (err, files) {
          if (err) {
            that.emit('error', err);
          }
          that.counter = that.counter + files.length - 1;
          files.forEach(function (filename) {
            that.getList(path.join(_path, filename));
          });
          that.emit('walk', path.resolve(_path), isDirectory);
        });
      }
    } else {
      that.counter = that.counter - 1;
      that.emit('walk', path.resolve(_path), isDirectory);
    }
  });
};

// 遍历方法
Walker.prototype.walk = function (source, callback) {
  if (typeof callback === 'function') {
    this.on('walk', callback);
  }
  this.getList(source);
};

module.exports = Walker;
{'fraction_non_alphanumeric': 0.16523994811932555, 'fraction_numerical': 0.004669260700389105, 'mean_word_length': 2.452103849597135, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3336350', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1340, 'n_words': 253}