<commit_before>"use strict";

var async = require('async');
var path = require("path");
var fs = require("fs");


var getDeletedFiles = function(oldCursor, newCursor) {
  var filesToDelete = [];

  for(var file in oldCursor) {
    if(!newCursor[file]) {
      filesToDelete.push(file);
    }
  }

  return filesToDelete;
};

var getCursorFromDirectory = function(dir, cb) {
  // Generate an object with files of the directory passed as argument
  // {
  //    '/a/file/somewhere': lastUpdateOfThisFile,
  //    '/another/file/somewhere': anotherDate
  // }
  var cursor = {};

  var queue = async.queue(function(task, cb) {
    async.waterfall([
      function readDir(cb) {
        fs.readdir(task.dir, cb);
      },
      function(files, cb) {
        async.map(files, function(nameOfFile, cb) {
          var file = path.join(task.dir, nameOfFile);
          // Recursive call for directory
          fs.lstat(file, function(err, res) {
            if(res.isDirectory()) {
              queue.push({'dir' : file, 'base' : task.base + nameOfFile + '/'});
              return cb(err);
            }
            else {
              fs.stat(file, function(err, stats) {
                if(err) {
                  return cb(err);
                }

                // Skip empty files
                if(stats.size !== 0) {
                  cursor[task.base + nameOfFile] = stats.mtime.getTime();
                }

                cb();
              });
            }
          });
        }, cb);
      }
    ], cb);
  }, 3);

  // Push first item to start pseudo-recursion
  queue.push({'dir' : dir, 'base' : '/'});
<commit_msg>print files when checking files<commit_after>"use strict";

var async = require('async');
var path = require("path");
var fs = require("fs");


var print = function(string) {
  process.stdout.clearLine();
  process.stdout.cursorTo(0);
  process.stdout.write("Listing files : ");
  process.stdout.write(string);
};

var getDeletedFiles = function(oldCursor, newCursor) {
  var filesToDelete = [];

  for(var file in oldCursor) {
    if(!newCursor[file]) {
      filesToDelete.push(file);
    }
  }

  return filesToDelete;
};

var getCursorFromDirectory = function(dir, cb) {
  // Generate an object with files of the directory passed as argument
  // {
  //    '/a/file/somewhere': lastUpdateOfThisFile,
  //    '/another/file/somewhere': anotherDate
  // }
  var cursor = {};

  var queue = async.queue(function(task, cb) {
    async.waterfall([
      function readDir(cb) {
        fs.readdir(task.dir, cb);
      },
      function(files, cb) {
        async.map(files, function(nameOfFile, cb) {
          var file = path.join(task.dir, nameOfFile);
          // Recursive call for directory
          fs.lstat(file, function(err, res) {
            if(res.isDirectory()) {
              queue.push({'dir' : file, 'base' : task.base + nameOfFile + '/'});
              return cb(err);
            }
            else {
              fs.stat(file, function(err, stats) {
                if(err) {
                  return cb(err);
                }

                // Skip empty files
                if(stats.size !== 0) {
                  cursor[task.base + nameOfFile] = stats.mtime.getTime();
                  print(nameOfFile);
                }

                cb();
              });
            }
          });
        }, cb);
      }
    ], cb);
  }, 3);

  // Push first item to start pseudo-recursion
  queue.push({'dir' : dir, 'base' : '/'});

{'fraction_non_alphanumeric': 0.14596451058958215, 'fraction_numerical': 0.0014310246136233543, 'mean_word_length': 1.7870813397129186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2851610', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1037, 'n_words': 227}