<commit_before>#!/usr/bin/env python

import argparse
import unicodecsv as csv
import json
import logging
import locale
import StringIO
from struct import Struct

from messytables import (any_tableset, headers_guess, headers_processor,
                         offset_processor, type_guess, types_processor)
from messytables import (StringType, IntegerType, DecimalType)

import myria

# Set the log level here
logging.getLogger().setLevel(logging.INFO)


def pretty_json(obj):
    return json.dumps(obj, indent=4, separators=(',', ': '))


def parse_args(argv=None):
    """Parse the arguments for this program"""
    parser = argparse.ArgumentParser(description='Upload a plaintext dataset to Myria',  # noqa
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)   # noqa

    # Hostname
    parser.add_argument('--hostname', '-n', help="Myria REST server hostname",
                        default='rest.myria.cs.washington.edu')

    # Port
    def check_valid_port(p):
        "True if p is a valid port number"
        try:
            p = int(p)
            assert p > 0 and p < 65536
            return p
        except:
            raise argparse.ArgumentTypeError('invalid port [1, 65535]: %s' % p)

    def set_locale(name):
        try:
            locale.setlocale(
                locale.LC_NUMERIC, '{name}.UTF-8'.format(name=name))
        except:
            raise argparse.ArgumentTypeError('invalid locale: %s' % name)
        else:
            return name

    parser.add_argument('--port', '-p', help="Myria REST server port",
                        default=1776, type=check_valid_port)

    parser.add_argument('file', help="File to be uploaded",
                        type=argparse.FileType('rb'))

    parser.add_argument('--user', help="User who owns the relation",
                        type=str, default="public")
    parser.add_argument('--program', help="Program that owns the relation",
                        type=str, default="adhoc")
    parser.add_argument('--relation', help="Relation name",
                        type=str, required=True)
    parser.add_argument('--locale', '-l',
                        help="locale to improve number guessing",
                        type=set_locale)

    parser.add_argument('--overwrite', '-o', help="Overwrite existing data",
                        action='store_true')

    return parser.parse_args(argv)


def convert_type(type_):
    "Convert a MessyTables type to a Myria type."
    if isinstance(type_, StringType):
        return "STRING_TYPE"
    elif isinstance(type_, IntegerType):<commit_msg>Use stdin if no path is provided, Fixes #14<commit_after>#!/usr/bin/env python

import argparse
import json
import logging
import locale
import sys
import StringIO
from struct import Struct

import unicodecsv as csv
from messytables import (any_tableset, headers_guess, headers_processor,
                         offset_processor, type_guess, types_processor)
from messytables import (StringType, IntegerType, DecimalType)

import myria

# Set the log level here
logging.getLogger().setLevel(logging.INFO)


def pretty_json(obj):
    return json.dumps(obj, indent=4, separators=(',', ': '))


def parse_args(argv=None):
    """Parse the arguments for this program"""
    parser = argparse.ArgumentParser(description='Upload a plaintext dataset to Myria',  # noqa
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)   # noqa

    # Hostname
    parser.add_argument('--hostname', '-n', help="Myria REST server hostname",
                        default='rest.myria.cs.washington.edu')

    # Port
    def check_valid_port(p):
        "True if p is a valid port number"
        try:
            p = int(p)
            assert p > 0 and p < 65536
            return p
        except:
            raise argparse.ArgumentTypeError('invalid port [1, 65535]: %s' % p)

    def set_locale(name):
        try:
            locale.setlocale(
                locale.LC_NUMERIC, '{name}.UTF-8'.format(name=name))
        except:
            raise argparse.ArgumentTypeError('invalid locale: %s' % name)
        else:
            return name

    parser.add_argument('--port', '-p', help="Myria REST server port",
                        default=1776, type=check_valid_port)

    parser.add_argument('file', help="File to be uploaded",
                        default=sys.stdin, nargs='?',
                        type=argparse.FileType('rb'))

    parser.add_argument('--user', help="User who owns the relation",
                        type=str, default="public")
    parser.add_argument('--program', help="Program that owns the relation",
                        type=str, default="adhoc")
    parser.add_argument('--relation', help="Relation name",
                        type=str, required=True)
    parser.add_argument('--locale', '-l',
                        help="locale to improve number guessing",
                        type=set_locale)

    parser.add_argument('--overwrite', '-o', help="Overwrite existing data",
                        action='store_true')

    return parser.parse_args(argv)


def convert_type(type_):
    "Convert a MessyTables type to a Myria type."
    if isinstance(type_, StringType):
        return "STRING_TYPE"
    elif isinstance(type_, IntegerType):
{'fraction_non_alphanumeric': 0.10813844714686623, 'fraction_numerical': 0.007109448082319925, 'mean_word_length': 2.34125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6404080', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1427, 'n_words': 389}