<commit_before># encoding: utf8
from optparse import OptionParser
import sys

from .db import connect, metadata
import pokedex.db.load
import pokedex.lookup

def main():
    if len(sys.argv) <= 1:
        command_help()

    command = sys.argv[1]
    args = sys.argv[2:]

    # XXX there must be a better way to get Unicode argv
    # XXX this doesn't work on Windows durp
    enc = sys.stdin.encoding
    args = [_.decode(enc) for _ in args]

    # Find the command as a function in this file
    func = globals().get("command_%s" % command, None)
    if func:
        func(*args)
    else:
        command_help()


def command_dump(*args):
    parser = OptionParser()
    parser.add_option('-e', '--engine', dest='engine_uri', default=None)
    parser.add_option('-d', '--directory', dest='directory', default=None)
    parser.add_option('-q', '--quiet', dest='verbose', default=True, action='store_false')
    options, _ = parser.parse_args(list(args))

    session = connect(options.engine_uri)
    pokedex.db.load.dump(session, directory=options.directory,
                                  verbose=options.verbose)

def command_load(*args):
    parser = OptionParser()
    parser.add_option('-e', '--engine', dest='engine_uri', default=None)
    parser.add_option('-d', '--directory', dest='directory', default=None)
    parser.add_option('-D', '--drop-tables', dest='drop_tables', default=False, action='store_true')<commit_msg>Fix crash when stdin has no encoding.<commit_after># encoding: utf8
from optparse import OptionParser
import sys

from .db import connect, metadata
import pokedex.db.load
import pokedex.lookup

def main():
    if len(sys.argv) <= 1:
        command_help()

    command = sys.argv[1]
    args = sys.argv[2:]

    # XXX there must be a better way to get Unicode argv
    # XXX this doesn't work on Windows durp
    enc = sys.stdin.encoding or 'utf8'
    args = [_.decode(enc) for _ in args]

    # Find the command as a function in this file
    func = globals().get("command_%s" % command, None)
    if func:
        func(*args)
    else:
        command_help()


def command_dump(*args):
    parser = OptionParser()
    parser.add_option('-e', '--engine', dest='engine_uri', default=None)
    parser.add_option('-d', '--directory', dest='directory', default=None)
    parser.add_option('-q', '--quiet', dest='verbose', default=True, action='store_false')
    options, _ = parser.parse_args(list(args))

    session = connect(options.engine_uri)
    pokedex.db.load.dump(session, directory=options.directory,
                                  verbose=options.verbose)

def command_load(*args):
    parser = OptionParser()
    parser.add_option('-e', '--engine', dest='engine_uri', default=None)
    parser.add_option('-d', '--directory', dest='directory', default=None)
    parser.add_option('-D', '--drop-tables', dest='drop_tables', default=False, action='store_true')
{'fraction_non_alphanumeric': 0.14428374655647383, 'fraction_numerical': 0.0030991735537190084, 'mean_word_length': 3.4015151515151514, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1832857', 'n_tokens_mistral': 919, 'n_tokens_neox': 894, 'n_words': 231}