<commit_before>#! /usr/bin/env node
'use strict';

(function () {

  var yargs = require('yargs');
  var packageJson = require('../package.json');
  var version = packageJson.version;

  var argv = yargs
  .usage('Usage: $0 <command> [options]')
  .command('crawl', 'Crawl a domain')
  .example('$0 crawl domain.com --depth 100', '(Crawl a domain to a depth of 100)')
  .alias('crawl', 'c')
  .option(
    'url',
    {
      group: 'crawl options:',
      alias : 'u',
      required: true,
      type: 'string',
      describe: 'URL to crawl'
    }
  )
  .option(
    'depth',
    {
      group: 'crawl options:',
      alias : 'd',
      default : 10,
      type: 'number',
      describe: 'Depth to crawl'
    }
  )
  .option(
    'format',
    {
      group: 'crawl options:',
      alias : 'f',
      default : 'json',
      type: 'string',
      describe: 'Format to return',
      choices: ['json']
    }
  )
  .help('help')
  .alias('help', 'h')
  .version('version', 'Return the version number', version)
  .alias('version', 'v')
  .demand(1)
  .strict()
  .argv;

  console.log(argv);

})();<commit_msg>Move crawl options to crawl command<commit_after>#! /usr/bin/env node
'use strict';

(function () {

  var yargs = require('yargs');
  var packageJson = require('../package.json');
  var version = packageJson.version;

  var argv = yargs
  .usage('Usage: $0 <command> [options]')
  .command('crawl', 'Crawl a domain')
  .example('$0 crawl domain.com --depth 100', '(Crawl a domain to a depth of 100)')
  .alias('crawl', 'c')
  .command('crawl', 'Crawl a domain', {
    url: {
      alias : 'u',
      required: true,
      type: 'string',
      describe: 'URL to crawl'
    },
    depth: {
      alias : 'd',
      default : 10,
      type: 'number',
      describe: 'Depth to crawl'
    },
    format: {
      alias : 'f',
      default : 'json',
      type: 'string',
      describe: 'Format to return',
      choices: ['json']
    }
  })
  .help('help')
  .alias('help', 'h')
  .version('version', 'Return the version number', version)
  .alias('version', 'v')
  .demand(1)
  .strict()
  .argv;

  console.log(argv);

})();
{'fraction_non_alphanumeric': 0.18747078073866294, 'fraction_numerical': 0.010285179990649837, 'mean_word_length': 2.46839546191248, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1239392', 'n_tokens_mistral': 788, 'n_tokens_neox': 745, 'n_words': 203}