<commit_before>import optparse
import logging
import sys
from unfurl import Crawler, Page
import sqlite3

LOG = logging.getLogger(__name__)

def error(message):
    sys.stderr.write(' unfurl: error: %s\n' % message)

def abort(message):
    error(message)
    raise SystemExit(1)

def get_cli():
    cli = optparse.OptionParser(prog='unfurl',
      usage='unfurl <subcommand> [arguments] [options]',
      epilog='Valid subcommands: %s' % ', '.join(COMMANDS.keys()))
    cli.add_option('--debug', action='store_true',
      help='enable debug output')
    return cli

def get_crawl_cli():
    cli = optparse.OptionParser(prog='unfurl crawl')
    cli.add_option('-p', '--period', type=int, default=3600,
      help='Time in seconds between crawls. Defaults to 1hr')
    cli.add_option('-t', '--times', type=int, default=-1,
      help='Crawl a specified number of times (defaults to forever)')
    return cli

def main_main(argv):
    cli = get_cli()
    opts, args = cli.parse_args(['-h'])

def main_diff(argv):
    pass

def main_crawl(argv):
    cli = get_crawl_cli()
    opts, args = cli.parse_args(argv)

    if args:
        LOG.debug('command-line set to crawl pages: %s' % ', '.join(args))

    if opts.period < 0:
        cli.error('must pass a positive period')

    crawler = Crawler(period=opts.period, count=opts.times)
    pages = [ Page(i) for i in args ]
    crawler.crawl(pages)
<commit_msg>Instrument config into CLI<commit_after>import optparse
import logging
import sys
from unfurl import Crawler, Page
from unfurl.config import CONFIG, ConfigurationError
import sqlite3

LOG = logging.getLogger(__name__)

def error(message):
    sys.stderr.write(' unfurl: error: %s\n' % message)

def abort(message):
    error(message)
    raise SystemExit(1)

def get_cli():
    cli = optparse.OptionParser(prog='unfurl',
      usage='unfurl <subcommand> [arguments] [options]',
      epilog='Valid subcommands: %s' % ', '.join(COMMANDS.keys()))
    cli.add_option('--debug', action='store_true',
      help='enable debug output')
    return cli

def get_crawl_cli():
    cli = optparse.OptionParser(prog='unfurl crawl')
    cli.add_option('-p', '--period', type=int, default=3600,
      help='Time in seconds between crawls. Defaults to 1hr')
    cli.add_option('-t', '--count', type=int, default=-1,
      help='Crawl a specified number of times (defaults to forever)')
    return cli

def main_main(argv):
    cli = get_cli()
    opts, args = cli.parse_args(['-h'])

def main_diff(argv):
    pass

def load_config(cli, config=None):
    config = config or CONFIG
    try:
        config.load()
    except ConfigurationError, e:
        cli.error('problem understanding configuration: ' + e.exception.args[0])        

def main_crawl(argv):
    cli = get_crawl_cli()
    opts, args = cli.parse_args(argv)

    if args:
        LOG.debug('command-line set to crawl pages: %s' % ', '.join(args))

    load_config(cli)

    crawler = Crawler(
      period=CONFIG.get('crawler', 'period'),
      count=CONFIG.get('crawler', 'count'),
      threaded=CONFIG.get('crawler', 'threaded'),
      max_threads=CONFIG.get('crawler', 'max_threads')
    )
    pages = [ Page(i) for i in args ]
    crawler.crawl(pages)

{'fraction_non_alphanumeric': 0.14227895392278955, 'fraction_numerical': 0.00560398505603985, 'mean_word_length': 3.5509915014164304, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6222106', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1070, 'n_words': 268}