<commit_before>// @flow

import childProcess from 'child_process';
import glob from 'glob';

import formattedToRegex from './formattedToRegex';

function findMatchingFilesWithFind(
  lookupPath: string,
  validFilesRegex: string
): Array<string> {
  const findCommand = [
    `find ${lookupPath}`,
    '-name "**.js*"',
    '-not -path "./node_modules/*"',
  ].join(' ');
  const command = `${findCommand} | egrep -i \"${validFilesRegex}\"`;

  // TODO switch to spawn so we can start processing the stream as it comes
  // in.
  let out = '';
  let err = '';
  try {
    out = String(childProcess.execSync(command));
  } catch (error) {
    err = String(error.stderr);
  }

  if (err !== '') {
    throw new Error(err);
  }

  // Filter out empty lines before returning
  return out.split('\n').filter((file: string): boolean => !!file);
}

function findMatchingFilesWithNode(
  lookupPath: string,
  validFilesRegex: string
): Array<string> {
  return glob.sync(`${lookupPath}/**/*.js*`, {
    ignore: './node_modules/**',
  }).filter((filePath: string): bool =>
    new RegExp(validFilesRegex, 'i').test(filePath));
}

/**
 * Finds files from the local file system matching the variable name.
 */
export default function findMatchingFiles(
  lookupPath: string,
  variableName: string
): Array<string> {
  if (lookupPath === '') {
    // If lookupPath is an empty string, the `find` command will not work
    // as desired so we bail early.
    throw new Error(`lookup path cannot be empty (${lookupPath})`);
  }

  const formattedVarName = formattedToRegex(variableName);
  const validFilesRegex = `(/|^)${formattedVarName}(/index)?(/package)?\\.js.*`;

  if (/^win/.test(process.platform)) {
    return findMatchingFilesWithNode(lookupPath, validFilesRegex);
  }
  return findMatchingFilesWithFind(lookupPath, validFilesRegex);
}<commit_msg>Allow finder to be controlled through an environment variable<commit_after>// @flow

import childProcess from 'child_process';
import glob from 'glob';

import formattedToRegex from './formattedToRegex';

function findMatchingFilesWithFind(
  lookupPath: string,
  validFilesRegex: string
): Array<string> {
  const findCommand = [
    `find ${lookupPath}`,
    '-name "**.js*"',
    '-not -path "./node_modules/*"',
  ].join(' ');
  const command = `${findCommand} | egrep -i \"${validFilesRegex}\"`;

  // TODO switch to spawn so we can start processing the stream as it comes
  // in.
  let out = '';
  let err = '';
  try {
    out = String(childProcess.execSync(command));
  } catch (error) {
    err = String(error.stderr);
  }

  if (err !== '') {
    throw new Error(err);
  }

  // Filter out empty lines before returning
  return out.split('\n').filter((file: string): boolean => !!file);
}

function findMatchingFilesWithNode(
  lookupPath: string,
  validFilesRegex: string
): Array<string> {
  return glob.sync(`${lookupPath}/**/*.js*`, {
    ignore: './node_modules/**',
  }).filter((filePath: string): bool =>
    new RegExp(validFilesRegex, 'i').test(filePath));
}

/**
 * Finds files from the local file system matching the variable name.
 */
export default function findMatchingFiles(
  lookupPath: string,
  variableName: string
): Array<string> {
  if (lookupPath === '') {
    // If lookupPath is an empty string, the `find` command will not work
    // as desired so we bail early.
    throw new Error(`lookup path cannot be empty (${lookupPath})`);
  }

  const formattedVarName = formattedToRegex(variableName);
  const validFilesRegex = `(/|^)${formattedVarName}(/index)?(/package)?\\.js.*`;

  if (/^win/.test(process.platform) || process.env.IMPORT_JS_USE_NODE_FINDER) {
    return findMatchingFilesWithNode(lookupPath, validFilesRegex);
  }
  return findMatchingFilesWithFind(lookupPath, validFilesRegex);
}
{'fraction_non_alphanumeric': 0.15392753239883628, 'fraction_numerical': 0.0, 'mean_word_length': 3.9308996088657104, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7186938', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1145, 'n_words': 327}