<commit_before>#!/usr/bin/env node

/**
 * Copyright (C) 2014 Glayzzle (BSD3 License)
 * @authors https://github.com/glayzzle/php-parser/graphs/contributors
 * @url http://glayzzle.com
 */

var util    = require('util');
var fs      = require('fs');
var path    = require('path');

var engine  = require('../lib/src/index');
engine = new engine();
engine.lexer.short_tags = true;

// help screen
function printHelp() {
  console.log('Usage: test [options] [-f] <file>');
  console.log('');
  console.log('  -f <file>                      Parse and test the specified file');
  console.log('  -d <path>                      Parse each file in the specified path');
  console.log('  -m <path>                      Run mocha tests on the specified path');
  console.log('  -r                             Use recursivity with the specified path');
  console.log('  -e                             Eval the specified input and shows AST');
  console.log('  -v                             Enable verbose mode and show debug');
  console.log('  -s                             Silent error mode, and try to suppress errors');
  console.log('  --asp_short                    Test with short tags');
  console.log('  -h, --help                     Print help and exit');
}


// aborts the execution with the specified error message
function abort(message) {
  console.error(message);
  process.exit(1);
}

/* Arguments */
var options = {
  filename: null,
  path: null,
  recursive: false,
  evalCode: false,
  aspShort: false,
  mocha: false
};

var args = process.argv.slice(2); // Trim 'node' and the script path.


function isOption(arg) {
  return (/^-/).test(arg);
}

function nextArg() {
  args.shift();
}

// Reading arguments
while (args.length > 0 && isOption(args[0])) {
  switch(args[0]) {
    case '-f':
      nextArg();
      options.filename = args[0];
      break;

    case '-e':
      nextArg();
      options.evalCode = args[0];
      break;

    case '--asp_short':
      engine.lexer.asp_tags = true;
      options.aspShort = true;
      break;

    case '--debug':
    case '-v':
      engine.parser.locations = true;
      engine.parser.debug = true;
      break;
<commit_msg>add more options<commit_after>#!/usr/bin/env node

/**
 * Copyright (C) 2014 Glayzzle (BSD3 License)
 * @authors https://github.com/glayzzle/php-parser/graphs/contributors
 * @url http://glayzzle.com
 */

var util    = require('util');
var fs      = require('fs');
var path    = require('path');

/**
 * Initialize engine
 */
var engine  = require('../lib/src/index');
engine = new engine({
  ast: {
    withPositions: true
  },
  lexer: {
    short_tags: true
  },
  parser: {
    extractDoc: true
  }
});

// help screen
function printHelp() {
  console.log('Usage: test [options] [-f] <file>');
  console.log('');
  console.log('  -f <file>                      Parse and test the specified file');
  console.log('  -d <path>                      Parse each file in the specified path');
  console.log('  -m <path>                      Run mocha tests on the specified path');
  console.log('  -r                             Use recursivity with the specified path');
  console.log('  -e                             Eval the specified input and shows AST');
  console.log('  -v                             Enable verbose mode and show debug');
  console.log('  -s                             Silent error mode, and try to suppress errors');
  console.log('  --asp_short                    Test with short tags');
  console.log('  -h, --help                     Print help and exit');
}


// aborts the execution with the specified error message
function abort(message) {
  console.error(message);
  process.exit(1);
}

/* Arguments */
var options = {
  filename: null,
  path: null,
  recursive: false,
  evalCode: false,
  aspShort: false,
  mocha: false
};

var args = process.argv.slice(2); // Trim 'node' and the script path.


function isOption(arg) {
  return (/^-/).test(arg);
}

function nextArg() {
  args.shift();
}

// Reading arguments
while (args.length > 0 && isOption(args[0])) {
  switch(args[0]) {
    case '-f':
      nextArg();
      options.filename = args[0];
      break;

    case '-e':
      nextArg();
      options.evalCode = args[0];
      break;

    case '--asp_short':
      engine.lexer.asp_tags = true;
      options.aspShort = true;
      break;

    case '--debug':
    case '-v':
      engine.parser.locations = true;
      engine.parser.debug = true;
      engine.lexer.debug = true;
      break;

{'fraction_non_alphanumeric': 0.14323607427055704, 'fraction_numerical': 0.005305039787798408, 'mean_word_length': 2.074048913043478, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6662581', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1356, 'n_words': 401}