<commit_before>/*global fs:false, path:false */

/* exported requestAsync */
function requestAsync(url, callback) {
  var req = new XMLHttpRequest();

  req.addEventListener('load', function() {
    callback(this.status, req.responseText);
  });
  req.addEventListener('error', function() {
    callback(this.status);
  });

  req.open('GET', url, true);
  req.send();

  return req;
}

/* exported browseDirectories */
function browseDirectories(paths, cb, step) {
  var total = paths.length;
  var files = [];

  var isExt = new RegExp(JSON.parse(localStorage.extentions).join('|'), 'i');

  var handler = function(err, f) {
    if (!err) {
      files = f.concat(files);
    }
    if (--total === 0) {
      cb(files);
    }
  };

  for (var p = paths.length - 1; p >= 0; p--) {
    browseDirectory(paths[p], isExt, handler, step);
  }
}

function browseDirectory(dir, isExt, done, step) {
  if (step !== undefined) {
    step(dir);
  }
  var files = [];
  fs.readdir(dir, function(err, list) {
    if (err) {
      return done(err);
    }
    var pending = list.length;
    if (!pending) {
      return done(null, files);
    }
    list.map(function(file) {
      file = path.resolve(dir, file);
      fs.stat(file, function(err, stat) {
        if (stat && stat.isDirectory()) {
          browseDirectory(file, isExt, function(err, res) {
            files = files.concat(res);
            if (!--pending) {
              done(null, files);
            }
          }, step);
        } else {
          if (isExt.test(path.extname(file))) {
            files.push(file);
          }<commit_msg>Fix error of access denied scandirs<commit_after>/*global fs:false, path:false */

/* exported requestAsync */
function requestAsync(url, callback) {
  var req = new XMLHttpRequest();

  req.addEventListener('load', function() {
    callback(this.status, req.responseText);
  });
  req.addEventListener('error', function() {
    callback(this.status);
  });

  req.open('GET', url, true);
  req.send();

  return req;
}

/* exported browseDirectories */
function browseDirectories(paths, cb, step) {
  var total = paths.length;
  var files = [];

  var isExt = new RegExp(JSON.parse(localStorage.extentions).join('|'), 'i');

  var handler = function(err, f) {
    if (!err) {
      files = f.concat(files);
    } else {
      console.error(err.message);
    }
    if (!--total) {
      cb(files);
    }
  };

  for (var p = paths.length - 1; p >= 0; p--) {
    browseDirectory(paths[p], isExt, handler, step);
  }
}

function browseDirectory(dir, isExt, done, step) {
  if (step !== undefined) {
    step(dir);
  }
  var files = [];
  fs.readdir(dir, function(err, list) {
    if (err) {
      return done(err);
    }
    var pending = list.length;
    if (!pending) {
      return done(null, files);
    }
    list.map(function(file) {
      file = path.resolve(dir, file);
      fs.stat(file, function(err, stat) {
        if (stat && stat.isDirectory()) {
          browseDirectory(file, isExt, function(err, res) {
            if (!err) {
              files = files.concat(res);
            } else {
              console.error(err.message);
            }
            if (!--pending) {
              done(null, files);
            }
          }, step);
        } else {
          if (err) {
            console.error(err.message);
          }
          if (isExt.test(path.extname(file))) {
            files.push(file);
          }
{'fraction_non_alphanumeric': 0.1629802095459837, 'fraction_numerical': 0.0014551804423748546, 'mean_word_length': 2.264007597340931, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '459974', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1096, 'n_words': 245}