<commit_before>#!/usr/bin/env node

var program = require('commander')
  , fs = require('fs')
  , winston = require('winston')
  , which = require('which').sync
  , path = require('path')
  , thresher = require('thresher')
  , Thresher = thresher.Thresher
  , ScraperBox = thresher.ScraperBox
  , Scraper = thresher.Scraper
  , ep = require('../lib/eventparse.js')
  , loglevels = require('../lib/loglevels.js')
  , outformat = require('../lib/outformat.js');


var pjson = require('../package.json');
QSVERSION =  pjson.version;

program
  .version(pjson.version)
  .option('-u, --url <url>',
          'URL to scrape')
  .option('-r, --urllist <path>',
          'path to file with list of URLs to scrape (one per line)')
  .option('-s, --scraper <path>',
          'path to scraper definition (in JSON format)')
  .option('-d, --scraperdir <path>',
          'path to directory containing scraper definitions (in JSON format)')
  .option('-o, --output <path>',
          'where to output results ' +
          '(directory will be created if it doesn\'t exist',
          'output')
  .option('-r, --ratelimit <int>',
          'maximum number of scrapes per minute (default 3)', 3)
  .option('-h --headless',
          'render all pages in a headless browser')
  .option('-l, --loglevel <level>',
          'amount of information to log ' +
          '(silent, verbose, info*, data, warn, error, or debug)',
          'info')
  .option('-f, --outformat <name>',
          'JSON format to transform results into (currently only bibjson)')
  .parse(process.argv);
<commit_msg>Change short arg for ratelimit to avoid conflict (fixes #53)<commit_after>#!/usr/bin/env node

var program = require('commander')
  , fs = require('fs')
  , winston = require('winston')
  , which = require('which').sync
  , path = require('path')
  , thresher = require('thresher')
  , Thresher = thresher.Thresher
  , ScraperBox = thresher.ScraperBox
  , Scraper = thresher.Scraper
  , ep = require('../lib/eventparse.js')
  , loglevels = require('../lib/loglevels.js')
  , outformat = require('../lib/outformat.js');


var pjson = require('../package.json');
QSVERSION =  pjson.version;

program
  .version(pjson.version)
  .option('-u, --url <url>',
          'URL to scrape')
  .option('-r, --urllist <path>',
          'path to file with list of URLs to scrape (one per line)')
  .option('-s, --scraper <path>',
          'path to scraper definition (in JSON format)')
  .option('-d, --scraperdir <path>',
          'path to directory containing scraper definitions (in JSON format)')
  .option('-o, --output <path>',
          'where to output results ' +
          '(directory will be created if it doesn\'t exist',
          'output')
  .option('-i, --ratelimit <int>',
          'maximum number of scrapes per minute (default 3)', 3)
  .option('-h --headless',
          'render all pages in a headless browser')
  .option('-l, --loglevel <level>',
          'amount of information to log ' +
          '(silent, verbose, info*, data, warn, error, or debug)',
          'info')
  .option('-f, --outformat <name>',
          'JSON format to transform results into (currently only bibjson)')
  .parse(process.argv);

{'fraction_non_alphanumeric': 0.1678649577993123, 'fraction_numerical': 0.001875586120662707, 'mean_word_length': 2.764705882352941, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3467909', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1042, 'n_words': 297}