# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels
# 
# Path: matcher.py
# class MultiSimilarityMatcher(object):
# 
#     def __init__(self):
#         self.hashtags = SimilarityMatcher()
#         self.words = SimilarityMatcher()
#         self.mentions = SimilarityMatcher()
# 
#     def update_from_dict(self, d):
#         words = {}
#         mentions = {}
#         tags = {}
# 
#         for w in d:
#             if w[0] == '@':
#                 mentions[w] = True
#             elif w[0] == '#':
#                 tags[w] = True
#             else:
#                 words[w] = True
# 
#         self.hashtags.update_from_dict(tags)
#         self.mentions.update_from_dict(mentions)
#         self.words.update_from_dict(words)
# 
#     def get_most_similar_word(self, w):
#         if w[0] == '@':
#             return self.mentions.get_most_similar_word(w)
#         elif w[0] == '#':
#             return self.hashtags.get_most_similar_word(w)
#         else:
#             return self.words.get_most_similar_word(w)
# 
#     def expand_dict(self, target, other):
#         self.update_from_dict(target)
# 
#         words_to_find = set([])
#         for w in other:
#             if w not in target:
#                 words_to_find.add(w)
# 
#         logging.info("We've got %d words to find", len(words_to_find))
# 
#         for w in words_to_find:
#             _, m = self.get_most_similar_word(w)
#             target[w] = target[m]
#             logging.debug("'%s' => '%s'", w, m)


from modelio import load_pos_tagged_data
from matcher import MultiSimilarityMatcher
import logging
import pickle

"""

    This file generates a substitution dictionary for the test dataset.

    The idea here is that if you can substitute a word for something that's
    fairly similar, it's better than just relying on the zero word.

"""

logging.basicConfig(level=logging.DEBUG)

with open('substitutions.pkl', 'wb') as fout:

    word_dict = {}
    load_pos_tagged_data("Data/Brown.conll", worddict=word_dict)
    load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

    test_dict = {}

# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels
# 
# Path: matcher.py
# class MultiSimilarityMatcher(object):
# 
#     def __init__(self):
#         self.hashtags = SimilarityMatcher()
#         self.words = SimilarityMatcher()
#         self.mentions = SimilarityMatcher()
# 
#     def update_from_dict(self, d):
#         words = {}
#         mentions = {}
#         tags = {}
# 
#         for w in d:
#             if w[0] == '@':
#                 mentions[w] = True
#             elif w[0] == '#':
#                 tags[w] = True
#             else:
#                 words[w] = True
# 
#         self.hashtags.update_from_dict(tags)
#         self.mentions.update_from_dict(mentions)
#         self.words.update_from_dict(words)
# 
#     def get_most_similar_word(self, w):
#         if w[0] == '@':
#             return self.mentions.get_most_similar_word(w)
#         elif w[0] == '#':
#             return self.hashtags.get_most_similar_word(w)
#         else:
#             return self.words.get_most_similar_word(w)
# 
#     def expand_dict(self, target, other):
#         self.update_from_dict(target)
# 
#         words_to_find = set([])
#         for w in other:
#             if w not in target:
#                 words_to_find.add(w)
# 
#         logging.info("We've got %d words to find", len(words_to_find))
# 
#         for w in words_to_find:
#             _, m = self.get_most_similar_word(w)
#             target[w] = target[m]
#             logging.debug("'%s' => '%s'", w, m)


# Path: substitution.py
from modelio import load_pos_tagged_data
from matcher import MultiSimilarityMatcher
import logging
import pickle


"""

    This file generates a substitution dictionary for the test dataset.

    The idea here is that if you can substitute a word for something that's
    fairly similar, it's better than just relying on the zero word.

"""

logging.basicConfig(level=logging.DEBUG)

with open('substitutions.pkl', 'wb') as fout:

    word_dict = {}
    load_pos_tagged_data("Data/Brown.conll", worddict=word_dict)
    load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

    test_dict = {}

    load_pos_tagged_data("Data/TweeboDaily547.conll", worddict=test_dict)