<commit_before>#!/usr/bin/env node
/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

var argv = require('commander');

function collect(val, memo) {
  memo.push(val);
  return memo;
}

var defaultExtensions = ['js', 'jsx'];
var defaultExclude = [];
var defaultIgnore = ['node_modules', '__tests__', '__mocks__'];

argv
    .usage('[path...] [options]')
    .description(
        'Extract meta information from React components.\n' +
        '  If a directory is passed, it is recursively traversed.')
    .option(
        '-o, --out <file>',
        'Store extracted information in the FILE')
    .option(
        '--pretty',
        'pretty print JSON')
    .option(
        '-x, --extension <extension>',
        'File extensions to consider. Repeat to define multiple extensions. Default: ' + JSON.stringify(defaultExtensions),
        collect,
        ['js', 'jsx'])
    .option(
        '-e, --exclude <path>',
        'Filename pattern to exclude. Default: ' + JSON.stringify(defaultExclude),
        collect,
        [])
    .option(
        '-i, --ignore <path>',
        'Folders to ignore. Default: ' + JSON.stringify(defaultIgnore),
        collect,
        ['node_modules', '__tests__', '__mocks__'])
    .option(
        '--resolver <resolver>',
        'Resolver name (findAllComponentDefinitions, findExportedComponentDefinition) or path to a module that exports ' +
        'a resolver. Default: findExportedComponentDefinition',
        'findExportedComponentDefinition')
    .arguments('<path>');

argv.parse(process.argv);

var async = require('async');
var dir = require('node-dir');
var fs = require('fs');
var parser = require('../dist/main');
var path = require('path');

var output = argv.out;
var paths = argv.args || [];
var extensions = new RegExp('\\.(?:' + argv.extension.join('|') + ')$');
var ignoreDir = argv.ignore;
var excludePatterns = argv.exclude;
var resolver;
var errorMessage;

if (argv.resolver) {
  try {
    // Look for built-in resolver
    resolver = require(`../dist/resolver/${argv.resolver}`).default;
  } catch(e) {
    if (e.code !== 'MODULE_NOT_FOUND') {
      throw e;
    }
    const resolverPath = path.resolve(process.cwd(), argv.resolver);
    try {
      // Look for local resolver
      resolver = require(resolverPath);
    } catch (e) {
      if (e.code !== 'MODULE_NOT_FOUND') {
        throw e;
      }
      // Will exit with this error message
      errorMessage = (
        `Unknown resolver: "${argv.resolver}" is neither a built-in resolver ` +
        `nor can it be found locally ("${resolverPath}")`
      );
    }
  }
}

function parse(source) {
  return parser.parse(source, resolver);<commit_msg>-e parameter now handle regular espression<commit_after>#!/usr/bin/env node
/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

var argv = require('commander');

function collect(val, memo) {
  memo.push(val);
  return memo;
}

var defaultExtensions = ['js', 'jsx'];
var defaultExclude = [];
var defaultIgnore = ['node_modules', '__tests__', '__mocks__'];

argv
    .usage('[path...] [options]')
    .description(
        'Extract meta information from React components.\n' +
        '  If a directory is passed, it is recursively traversed.')
    .option(
        '-o, --out <file>',
        'Store extracted information in the FILE')
    .option(
        '--pretty',
        'pretty print JSON')
    .option(
        '-x, --extension <extension>',
        'File extensions to consider. Repeat to define multiple extensions. Default: ' + JSON.stringify(defaultExtensions),
        collect,
        ['js', 'jsx'])
    .option(
        '-e, --exclude <path>',
        'Filename pattern to exclude. Default: ' + JSON.stringify(defaultExclude),
        collect,
        [])
    .option(
        '-i, --ignore <path>',
        'Folders to ignore. Default: ' + JSON.stringify(defaultIgnore),
        collect,
        ['node_modules', '__tests__', '__mocks__'])
    .option(
        '--resolver <resolver>',
        'Resolver name (findAllComponentDefinitions, findExportedComponentDefinition) or path to a module that exports ' +
        'a resolver. Default: findExportedComponentDefinition',
        'findExportedComponentDefinition')
    .arguments('<path>');

argv.parse(process.argv);

var async = require('async');
var dir = require('node-dir');
var fs = require('fs');
var parser = require('../dist/main');
var path = require('path');

var output = argv.out;
var paths = argv.args || [];
var extensions = new RegExp('\\.(?:' + argv.extension.join('|') + ')$');
var ignoreDir = argv.ignore;
var excludePatterns = argv.exclude;
var resolver;
var errorMessage;
var regexRegex = /^\/(.*)\/([igymu]{0,5})$/;
if (excludePatterns && excludePatterns.length === 1 && regexRegex.test(excludePatterns[0])){
  var match = excludePatterns[0].match(regexRegex);
    excludePatterns = new RegExp(match[1], match[2]);
}

if (argv.resolver) {
  try {
    // Look for built-in resolver
    resolver = require(`../dist/resolver/${argv.resolver}`).default;
  } catch(e) {
    if (e.code !== 'MODULE_NOT_FOUND') {
      throw e;
    }
    const resolverPath = path.resolve(process.cwd(), argv.resolver);
    try {
      // Look for local resolver
      resolver = require(resolverPath);
    } catch (e) {
      if (e.code !== 'MODULE_NOT_FOUND') {
        throw e;
      }
      // Will exit with this error message
      errorMessage = (
        `Unknown resolver: "${argv.resolver}" is neither a built-in resolver ` +
        `nor can it be found locally ("${resolverPath}")`
      );
    }
  }
}

function parse(source) {
  return parser.parse(source, resolver);
{'fraction_non_alphanumeric': 0.14761050399608547, 'fraction_numerical': 0.0024465829391616374, 'mean_word_length': 2.8638941398865785, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4358702', 'n_tokens_mistral': 1846, 'n_tokens_neox': 1831, 'n_words': 543}