<commit_before>var path = require('path');
var find = require('shelljs').find;

var displayMenu = require('./helpers/display-menu');
var openFile = require('./helpers/open-file');
var isValidPath = require('./helpers/is-valid-path');
var makeFileObject = require('./helpers/make-file-object');
var editor = process.env.EDITOR;

var filterOnType = require('./helpers/filters').filterOnType;
var filterOnDir = require('./helpers/filters').filterOnDir;
var filterOnSearchTerms= require('./helpers/filters').filterOnSearchTerms;

module.exports = function(rootDir, config, options) {
  var sources = (options.source) ? [options.source] : config.sourceDirs;

  var workingDirs = sources.map(function(dir) {
    return path.resolve(rootDir, dir);
  }).filter(isValidPath);

  if (!workingDirs.length) {
    console.error('Source dir(s) are invalid.');
    process.exit(1);
  }

  var searchTerms = options._;
  var dir = options.dir;
  var types = options.type || config.types;
  var isGUI = options.gui || !editor;

  var files = find(workingDirs);

  // filter on type (required)
  files = filterOnType(files, types);

  // filter on dir (optional)
  files = (dir) ? filterOnDir(files, sources, dir) : files;

  // make file object here since we might need it for a deep search
  files = files.map(function(file) {
    return makeFileObject(file, rootDir);
  });

  // filter on search terms (optional)
  files = (searchTerms.length) ? filterOnSearchTerms(files, searchTerms, options.apropos) : files;

  if (!files.length) {
    console.log('No files match your query');
    process.exit(1);<commit_msg>eliminate directories from potential pool of paths<commit_after>var path = require('path');
var find = require('shelljs').find;
var test = require('shelljs').test;

var displayMenu = require('./helpers/display-menu');
var openFile = require('./helpers/open-file');
var isValidPath = require('./helpers/is-valid-path');
var makeFileObject = require('./helpers/make-file-object');
var editor = process.env.EDITOR;

var filterOnType = require('./helpers/filters').filterOnType;
var filterOnDir = require('./helpers/filters').filterOnDir;
var filterOnSearchTerms= require('./helpers/filters').filterOnSearchTerms;

module.exports = function(rootDir, config, options) {
  var sources = (options.source) ? [options.source] : config.sourceDirs;

  var workingDirs = sources.map(function(dir) {
    return path.resolve(rootDir, dir);
  }).filter(isValidPath);

  if (!workingDirs.length) {
    console.error('Source dir(s) are invalid.');
    process.exit(1);
  }

  var searchTerms = options._;
  var dir = options.dir;
  var types = options.type || config.types;
  var isGUI = options.gui || !editor;

  // get a list of all files in the working dirs, but eliminate directories
  // just in case for some reason the user searches with a blank `type`
  var files = find(workingDirs).filter(function(item) {
    return test('-f', item);
  });

  // filter on type (required)
  files = filterOnType(files, types);

  // filter on dir (optional)
  files = (dir) ? filterOnDir(files, sources, dir) : files;

  // make file object here since we might need it for a deep search
  files = files.map(function(file) {
    return makeFileObject(file, rootDir);
  });

  // filter on search terms (optional)
  files = (searchTerms.length) ? filterOnSearchTerms(files, searchTerms, options.apropos) : files;

  // now process the results
  if (!files.length) {
    console.log('No files match your query');
    process.exit(1);
{'fraction_non_alphanumeric': 0.14367160775370583, 'fraction_numerical': 0.0011402508551881414, 'mean_word_length': 4.206231454005935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4640951', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1066, 'n_words': 289}