<commit_before>var sys = require('sys')
  , fs = require('fs')
  , path = require('path')
  , events = require('events')
  ;

function walk (dir, options, callback) { 
  if (!callback) {callback = options; options = {}}
  if (!callback.files) callback.files = {};
  if (!callback.pending) callback.pending = 0;
  callback.pending += 1;
  fs.readdir(dir, function (err, files) {
    if (err) return callback(err);
    callback.pending -= 1;
    files.forEach(function (f) {
      f = path.join(dir, f);
      callback.pending += 1;
      fs.stat(f, function (err, stat) {
        if (err) return callback(err)
        callback.pending -= 1;
        if (options.ignoreDotFiles && path.basename(f)[0] === '.') return;
        if (options.filter && options.filter(f)) return;
        callback.files[f] = stat;
        if (stat.isDirectory()) walk(f, options, callback);
        if (callback.pending === 0) callback(null, callback.files);
      })
    })
    if (callback.pending === 0) callback(null, callback.files);
  })
  if (callback.pending === 0) callback(null, callback.files);
}

exports.watchTree = function ( root, options, callback ) {
  if (!callback) {callback = options; options = {}}
  walk(root, function (err, files) {
    var fileWatcher = function (f) {
      fs.watchFile(f, options, function (c, p) {
        files[f] = c;
        if (!files[f].isDirectory()) callback(f, c, p);
        else {
          fs.readdir(f, function (err, nfiles) {
            if (err) return;
            nfiles.forEach(function (b) {
              var file = path.join(f, b);
              if (!files[file]) {
                fs.stat(file, function (err, stat) {
                  callback(file, stat, null);<commit_msg>Fixing duplicate notifications.<commit_after>var sys = require('sys')
  , fs = require('fs')
  , path = require('path')
  , events = require('events')
  ;

function walk (dir, options, callback) { 
  if (!callback) {callback = options; options = {}}
  if (!callback.files) callback.files = {};
  if (!callback.pending) callback.pending = 0;
  callback.pending += 1;
  fs.stat(dir, function (err, stat) {
    if (err) return callback(err);
    callback.files[dir] = stat;
    fs.readdir(dir, function (err, files) {
      if (err) return callback(err);
      callback.pending -= 1;
      files.forEach(function (f) {
        f = path.join(dir, f);
        callback.pending += 1;
        fs.stat(f, function (err, stat) {
          if (err) return callback(err)
          callback.pending -= 1;
          if (options.ignoreDotFiles && path.basename(f)[0] === '.') return;
          if (options.filter && options.filter(f)) return;
          callback.files[f] = stat;
          if (stat.isDirectory()) walk(f, options, callback);
          if (callback.pending === 0) callback(null, callback.files);
        })
      })
      if (callback.pending === 0) callback(null, callback.files);
    })
    if (callback.pending === 0) callback(null, callback.files);
  })
  
}
exports.watchTree = function ( root, options, callback ) {
  if (!callback) {callback = options; options = {}}
  walk(root, function (err, files) {
    var fileWatcher = function (f) {
      fs.watchFile(f, options, function (c, p) {
        // Check if anything actually changed in stat
        if (files[f] && !files[f].isDirectory() && c.nlink !== 0 && files[f].ino == c.ino) return;
        files[f] = c;
        if (!files[f].isDirectory()) callback(f, c, p);
        else {
          fs.readdir(f, function (err, nfiles) {
            if (err) return;
            nfiles.forEach(function (b) {
              var file = path.join(f, b);
              if (!files[file]) {
                fs.stat(file, function (err, stat) {
                  callback(file, stat, null);
{'fraction_non_alphanumeric': 0.16271637816245008, 'fraction_numerical': 0.005059920106524634, 'mean_word_length': 2.477777777777778, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6994410', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1153, 'n_words': 302}