<commit_before>var stderr = process.stderr,
    fs     = require('fs'),
    path   = require('path')

var TypeSystem  = require('./typesystem').TypeSystem,
    Parser      = require('./parser'),
    reportError = require('./util').reportError

// Parse command-line arguments
var argv = require('optimist')
    .usage('Usage: hb [command] [options]')
    .demand(1)
    .argv

// Utility function to read a file, parse it, walk it, and then return
// the finalized tree. On error it prints a nice error report and then
// calls `process.exit`.
function treeForFile (file) {
  var contents = fs.readFileSync(file).toString(),
      name     = path.basename(file),
      parser   = new Parser()

  parser.file = name
  // Parse and type-check the file; catch and report any errors
  try {
    var tree       = parser.parse(contents),
        typesystem = new TypeSystem()
    typesystem.walk(tree)
  } catch (err) {
    reportError(err)
    process.exit(1)
  }
  return tree
}// treeForFile

function parseAndWalk (file, code) {
  var parser     = new Parser(),
      typesystem = new TypeSystem()
  parser.file = file
  var tree = parser.parse(contents)
  typesystem.walk(tree)
  return tree
}

var commands = {
  inspect: function (args) {
    var file = args[0]
    if (!file) {
      stderr.write("Missing file to inspect\n")
      return process.exit(2)
    }
    var tree = treeForFile(args[0])
    // Print the resulting prettified AST
    tree.print()
    process.exit(0)
  },// inspect
}

function run () {
  var commandArg = argv._[0],
      otherArgs  = argv._.slice(1),
      command    = commands[commandArg]

  if (!command) {
    return stderr.write("Unrecognized command '"+commandArg+"'\n")
  }
  command(otherArgs)
}

module.exports = {run: run}
<commit_msg>Add compile command<commit_after>var stderr = process.stderr,
    fs     = require('fs'),
    path   = require('path')

var TypeSystem  = require('./typesystem').TypeSystem,
    Parser      = require('./parser'),
    reportError = require('./util').reportError

// Parse command-line arguments
var argv = require('optimist')
    .usage('Usage: hb [command] [options]')
    .demand(1)
    .argv

// Utility function to read a file, parse it, walk it, and then return
// the finalized tree. On error it prints a nice error report and then
// calls `process.exit`.
function treeForFile (file) {
  var contents = fs.readFileSync(file).toString(),
      name     = path.basename(file),
      parser   = new Parser()

  parser.file = name
  // Parse and type-check the file; catch and report any errors
  try {
    var tree       = parser.parse(contents),
        typesystem = new TypeSystem()
    typesystem.walk(tree)
  } catch (err) {
    reportError(err)
    process.exit(1)
  }
  return tree
}// treeForFile

function parseAndWalk (file, code) {
  var parser     = new Parser(),
      typesystem = new TypeSystem()
  parser.file = file
  var tree = parser.parse(contents)
  typesystem.walk(tree)
  return tree
}

function fileFromArgs (args, idx) {
  var file = args[idx]
  if (!file) {
    stderr.write("Missing file to inspect\n")
    return process.exit(2)
  }
  return file
}

var commands = {
  inspect: function (args) {
    var file = fileFromArgs(args, 0),
        tree = treeForFile(file)
    // Print the resulting prettified AST
    tree.print()
    process.exit(0)
  },// inspect
  compile: function (args) {
    var file = fileFromArgs(args, 0),
        tree = treeForFile(file)
    // Load the JavaScript compile target and print the compiled source
    require('./targets/javascript')
    process.stdout.write(tree.compile())
  }
}

function run () {
  var commandArg = argv._[0],
      otherArgs  = argv._.slice(1),
      command    = commands[commandArg]

  if (!command) {
    return stderr.write("Unrecognized command '"+commandArg+"'\n")
  }
  command(otherArgs)
}

module.exports = {run: run}

{'fraction_non_alphanumeric': 0.1288911757139182, 'fraction_numerical': 0.004116285052739903, 'mean_word_length': 2.8725099601593627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5157079', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1198, 'n_words': 346}