# Path: tag.py
# class Tag(object):
#     """Individual word-tag tuple"""
#     def __init__(self, word, tag):
#         """
#         Creates a word-tag tuple
#         :param word: The word to store
#         :param tag: The associated part-of-speech tag
#         """
#         self.word = word
#         self.tag = IDENT[tag]
# 
#     def get_readable_tag(self):
#         """
#         Returns what tag this is.
#         :return: A human-readable tag
#         """
#         return IDENT.keys()[self.tag]
# 
#     def __str__(self):
#         return "Tag(%s, %s)" % (self.word, self.get_readable_tag())
# 
# Path: util.py
# def download_file(url, outpath):
#     """
#         Download a file to a given location over HTTP
#     :param url: The HTTP location to fetch.
#     :param outpath: The location to save it to
#     """
#     # Don't bother re-downloading stuff
#     if os.path.exists(outpath):
#         logging.info("%s already exists, skipping...", outpath)
#         return
# 
#     logging.info("Downloading %s => '%s'", url, outpath)
#     r = requests.get(url)
#     with open(outpath, 'w') as fout:
#         fout.write(r.content)


import nltk
import logging
import cPickle as pickle
from argparse import ArgumentParser
from tag import Tag
from util import download_file
__author__ = 'rtownsend'


TWEEBO_OCT27_DL = "https://raw.githubusercontent.com/brendano/ark-tweet-nlp/master/data/twpos-data-v0.3/oct27.conll"
TWEEBO_DAILY547_DL = "https://raw.githubusercontent.com/brendano/ark-tweet-nlp/master/data/twpos-data-v0.3/daily547.conll"

def process_cmd_line():
    """Read command line arguments"""
    p = ArgumentParser("Download and pre-process command line arguments")
    p.add_argument("--brown", action="store_true")
    p.add_argument("--tokens", nargs="+")

    args = p.parse_args()

    if not args.brown and not args.tokens:
        raise ValueError("No corpora specified (see --usage)")

    return args

#BROWN_TAG_DICT = {
#    'AT' : 'D', # Determiner, i.e. 'THE'
#    'NP-TL': 'Z', # Proper noun i.e. 'FULTON'
#    ''
#}

def process_brown():
    """Read, convert and save the Brown corpus"""
    tags = []
    logging.info("Reading Brown corpus....")
    for word, tag in nltk.corpus.brown.tagged_words(tagset="universal"):

# Path: tag.py
# class Tag(object):
#     """Individual word-tag tuple"""
#     def __init__(self, word, tag):
#         """
#         Creates a word-tag tuple
#         :param word: The word to store
#         :param tag: The associated part-of-speech tag
#         """
#         self.word = word
#         self.tag = IDENT[tag]
# 
#     def get_readable_tag(self):
#         """
#         Returns what tag this is.
#         :return: A human-readable tag
#         """
#         return IDENT.keys()[self.tag]
# 
#     def __str__(self):
#         return "Tag(%s, %s)" % (self.word, self.get_readable_tag())
# 
# Path: util.py
# def download_file(url, outpath):
#     """
#         Download a file to a given location over HTTP
#     :param url: The HTTP location to fetch.
#     :param outpath: The location to save it to
#     """
#     # Don't bother re-downloading stuff
#     if os.path.exists(outpath):
#         logging.info("%s already exists, skipping...", outpath)
#         return
# 
#     logging.info("Downloading %s => '%s'", url, outpath)
#     r = requests.get(url)
#     with open(outpath, 'w') as fout:
#         fout.write(r.content)


# Path: corpus.py
import nltk
import logging
import cPickle as pickle
from argparse import ArgumentParser
from tag import Tag
from util import download_file

__author__ = 'rtownsend'


TWEEBO_OCT27_DL = "https://raw.githubusercontent.com/brendano/ark-tweet-nlp/master/data/twpos-data-v0.3/oct27.conll"
TWEEBO_DAILY547_DL = "https://raw.githubusercontent.com/brendano/ark-tweet-nlp/master/data/twpos-data-v0.3/daily547.conll"

def process_cmd_line():
    """Read command line arguments"""
    p = ArgumentParser("Download and pre-process command line arguments")
    p.add_argument("--brown", action="store_true")
    p.add_argument("--tokens", nargs="+")

    args = p.parse_args()

    if not args.brown and not args.tokens:
        raise ValueError("No corpora specified (see --usage)")

    return args

#BROWN_TAG_DICT = {
#    'AT' : 'D', # Determiner, i.e. 'THE'
#    'NP-TL': 'Z', # Proper noun i.e. 'FULTON'
#    ''
#}

def process_brown():
    """Read, convert and save the Brown corpus"""
    tags = []
    logging.info("Reading Brown corpus....")
    for word, tag in nltk.corpus.brown.tagged_words(tagset="universal"):

        t = Tag(word, tag)