<commit_before>#!/usr/bin/env node

exports.command = {
  description: 'glob the files in all projects',
  arguments: '<pattern>'
};

if (require.main !== module) {
  return;
}

var async = require('async');
var fs = require('fs');
var Glob = require('glob').Glob;
var path = require('path');
var program = require('commander');

var storage = require('../lib/storage.js');
var utilities = require('../lib/utilities.js');

// XXX: Better way to word this?
program.option('-d, --dot', 'match files beginning with a . by default');

program.option('-f, --files', 'only match files');
program.option('--directories', 'only match directories');

program._name = 'glob';
program.usage('<command>');
program.parse(process.argv);

var options = {
  dot: program.dot
};

var pattern = program.args.join(' ');

storage.setup(function () {
  var projects = storage.query({ directory: { $has: true } },
    { sortBy: function (project) { return project.name.toLowerCase(); } });

  async.eachSeries(projects, function (project, cbEach) {
    var glob = new Glob(path.join(utilities.expand(project.directory),
      pattern), options);

    glob.on('match', function (match) {
      if (/node_modules/.test(match)) {
        return;
      }

      if (/\/\.git\//.test(match)) {
        return;
      }

      fs.stat(match, function (err, stats) {
        if ((stats.isFile() && !program.directories) ||
          (stats.isDirectory() && !program.files)) {
          console.log(match);
        }
      });
    });

    glob.on('error', function (err) {
      console.log('Error globbing projects:', err);

      process.exit(1);
    });

    glob.on('end', function () {
      cbEach();
    });
  });
});<commit_msg>Fix error in glob caused by broken symlinks<commit_after>#!/usr/bin/env node

exports.command = {
  description: 'glob the files in all projects',
  arguments: '<pattern>'
};

if (require.main !== module) {
  return;
}

var async = require('async');
var fs = require('fs');
var Glob = require('glob').Glob;
var path = require('path');
var program = require('commander');

var storage = require('../lib/storage.js');
var utilities = require('../lib/utilities.js');

// XXX: Better way to word this?
program.option('-d, --dot', 'match files beginning with a . by default');

program.option('-f, --files', 'only match files');
program.option('--directories', 'only match directories');

program._name = 'glob';
program.usage('<command>');
program.parse(process.argv);

var options = {
  dot: program.dot
};

var pattern = program.args.join(' ');

storage.setup(function () {
  var projects = storage.query({ directory: { $has: true } },
    { sortBy: function (project) { return project.name.toLowerCase(); } });

  async.eachSeries(projects, function (project, cbEach) {
    var glob = new Glob(path.join(utilities.expand(project.directory),
      pattern), options);

    glob.on('match', function (match) {
      if (/node_modules/.test(match)) {
        return;
      }

      if (/\/\.git\//.test(match)) {
        return;
      }

      fs.stat(match, function (err, stats) {
        // Sometimes there are broken symlinks
        if (err && err.code === 'ENOENT') {
          return console.log(match);
        } else if (err) {
          throw err;
        }

        if ((stats.isFile() && !program.directories) ||
          (stats.isDirectory() && !program.files)) {
          console.log(match);
        }
      });
    });

    glob.on('error', function (err) {
      console.log('Error globbing projects:', err);

      process.exit(1);
    });

    glob.on('end', function () {
      cbEach();
    });
  });
});
{'fraction_non_alphanumeric': 0.1860529217199559, 'fraction_numerical': 0.0005512679162072767, 'mean_word_length': 2.9190064794816415, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1416361', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1116, 'n_words': 271}