<commit_before>#!/usr/bin/env node

/**
 * sankaku.js
 * <NAME> - 2015
 */

var program = require('commander');
var fs = require('fs');

var pkg = require('./package.json');
var main = require('./lib/main.js');

// Processing arguments
program
  .version(pkg.version)
  .usage(pkg.name + ' [options] file')
  .option('-s, --separate', 'Generate separate files per type')
  .parse(process.argv);

// Validating arguments
var filePath = program.args[program.args.length - 1];
if (!filePath) {
  throw 'Error: invalid path!';
}
console.log('Reading file ' + filePath + '...');

var content = null;
try {
  content = fs.readFileSync(filePath, {encoding: 'utf8'}); 
} catch(e) {
  throw 'Error loading file!';
}
console.log('File read successfully!');

// Processing
var m = main();

console.log('Initializing..');
m.initialize(content);

console.log('Retrieving AST...');
m.retrieveAst();
console.log('AST successfully retrieved! AST has ' + m.stats() + 'objects!');
<commit_msg>Update sankaku.js<commit_after>#!/usr/bin/env node

/**
 * sankaku.js
 * <NAME> - 2015
 */

var program = require('commander');
var fs = require('fs');

var pkg = require('./package.json');
var main = require('./lib/main.js');

// Processing arguments
program
  .version(pkg.version)
  .usage(pkg.name + ' [options] file')
  .option('-s, --separate', 'Generate separate files per type')
  .parse(process.argv);

// Validating arguments
var filePath = program.args[program.args.length - 1];
if (!filePath) {
  throw 'Error: invalid path!';
}

// Processing
var m = main();

console.log('Initializing from file ' + filePath + '...');
m.initialize(content);

console.log('Retrieving AST...');
m.retrieveAst();
console.log('AST successfully retrieved! AST has ' + m.stats() + 'objects!');
{'fraction_non_alphanumeric': 0.1961341671404207, 'fraction_numerical': 0.0062535531552018195, 'mean_word_length': 4.072046109510087, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7001216', 'n_tokens_mistral': 604, 'n_tokens_neox': 576, 'n_words': 143}