# Path: matcher.py
# class SimilarityMatcher(object):
# 
#     def __init__(self):
#         self.words = set([])
#         pass
# 
#     @classmethod
#     def _similarity(cls, diff, gap, a, b):
# 
#         mat = numpy.zeros((len(b)+1, len(a)+1))
# 
#         for i in range(len(a)):
#             for j in range(len(b)):
# 
# 
#                 d = max(0, mat[j, i] + diff(a[i], b[j]))
#                 d = max(d, mat[j + 1, i] + gap)
#                 d = max(d, mat[j, i + 1] + gap)
#                 mat [j + 1, i + 1] = d
# 
#         return numpy.max(mat), mat
# 
#     def get_most_similar_word(self, word):
#         best_score, best_match = 0, ""
#         for w in self.words:
#             sim, _ = self._similarity(char_diff, -1, w, word)
#             if sim > best_score:
#                 best_score = sim
#                 best_match = w
# 
#         return best_score, best_match
# 
#     def update_from_dict(self, d):
#         self.words.update(d.keys())
# 
# class MultiSimilarityMatcher(object):
# 
#     def __init__(self):
#         self.hashtags = SimilarityMatcher()
#         self.words = SimilarityMatcher()
#         self.mentions = SimilarityMatcher()
# 
#     def update_from_dict(self, d):
#         words = {}
#         mentions = {}
#         tags = {}
# 
#         for w in d:
#             if w[0] == '@':
#                 mentions[w] = True
#             elif w[0] == '#':
#                 tags[w] = True
#             else:
#                 words[w] = True
# 
#         self.hashtags.update_from_dict(tags)
#         self.mentions.update_from_dict(mentions)
#         self.words.update_from_dict(words)
# 
#     def get_most_similar_word(self, w):
#         if w[0] == '@':
#             return self.mentions.get_most_similar_word(w)
#         elif w[0] == '#':
#             return self.hashtags.get_most_similar_word(w)
#         else:
#             return self.words.get_most_similar_word(w)
# 
#     def expand_dict(self, target, other):
#         self.update_from_dict(target)
# 
#         words_to_find = set([])
#         for w in other:
#             if w not in target:
#                 words_to_find.add(w)
# 
#         logging.info("We've got %d words to find", len(words_to_find))
# 
#         for w in words_to_find:
#             _, m = self.get_most_similar_word(w)
#             target[w] = target[m]
#             logging.debug("'%s' => '%s'", w, m)
# 
# def char_diff(a, b):
#     if a == b:
#         return 2
#     if a.lower() == b.lower():
#         return 1 # It's a match
#     return -1
# 
# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels


import unittest
import logging
from matcher import SimilarityMatcher, MultiSimilarityMatcher, char_diff
from modelio import load_pos_tagged_data

        # So for whatever reason, the score matrix doesn't exactly match
        # the one provided by Wikipedia, chances are there's something wrong.
        score, sim = SimilarityMatcher._similarity(char_diff, -1.0, a, b)

        print score
        print sim

        self.assertEqual(12, score)

    def test_most_similar(self):

        word_dict = {}
        load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

        self.assertEquals(word_dict["I"], 1)

        sim = SimilarityMatcher()
        sim.update_from_dict(word_dict)

        score, word = sim.get_most_similar_word("iPod")

        self.assertEqual(word, "ipod")

    def test_expand_dictionary(self):

        word_dict = {}
        load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

        test_dict = {}

# Path: matcher.py
# class SimilarityMatcher(object):
# 
#     def __init__(self):
#         self.words = set([])
#         pass
# 
#     @classmethod
#     def _similarity(cls, diff, gap, a, b):
# 
#         mat = numpy.zeros((len(b)+1, len(a)+1))
# 
#         for i in range(len(a)):
#             for j in range(len(b)):
# 
# 
#                 d = max(0, mat[j, i] + diff(a[i], b[j]))
#                 d = max(d, mat[j + 1, i] + gap)
#                 d = max(d, mat[j, i + 1] + gap)
#                 mat [j + 1, i + 1] = d
# 
#         return numpy.max(mat), mat
# 
#     def get_most_similar_word(self, word):
#         best_score, best_match = 0, ""
#         for w in self.words:
#             sim, _ = self._similarity(char_diff, -1, w, word)
#             if sim > best_score:
#                 best_score = sim
#                 best_match = w
# 
#         return best_score, best_match
# 
#     def update_from_dict(self, d):
#         self.words.update(d.keys())
# 
# class MultiSimilarityMatcher(object):
# 
#     def __init__(self):
#         self.hashtags = SimilarityMatcher()
#         self.words = SimilarityMatcher()
#         self.mentions = SimilarityMatcher()
# 
#     def update_from_dict(self, d):
#         words = {}
#         mentions = {}
#         tags = {}
# 
#         for w in d:
#             if w[0] == '@':
#                 mentions[w] = True
#             elif w[0] == '#':
#                 tags[w] = True
#             else:
#                 words[w] = True
# 
#         self.hashtags.update_from_dict(tags)
#         self.mentions.update_from_dict(mentions)
#         self.words.update_from_dict(words)
# 
#     def get_most_similar_word(self, w):
#         if w[0] == '@':
#             return self.mentions.get_most_similar_word(w)
#         elif w[0] == '#':
#             return self.hashtags.get_most_similar_word(w)
#         else:
#             return self.words.get_most_similar_word(w)
# 
#     def expand_dict(self, target, other):
#         self.update_from_dict(target)
# 
#         words_to_find = set([])
#         for w in other:
#             if w not in target:
#                 words_to_find.add(w)
# 
#         logging.info("We've got %d words to find", len(words_to_find))
# 
#         for w in words_to_find:
#             _, m = self.get_most_similar_word(w)
#             target[w] = target[m]
#             logging.debug("'%s' => '%s'", w, m)
# 
# def char_diff(a, b):
#     if a == b:
#         return 2
#     if a.lower() == b.lower():
#         return 1 # It's a match
#     return -1
# 
# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels


# Path: test_matcher.py
import unittest
import logging
from matcher import SimilarityMatcher, MultiSimilarityMatcher, char_diff
from modelio import load_pos_tagged_data


        # So for whatever reason, the score matrix doesn't exactly match
        # the one provided by Wikipedia, chances are there's something wrong.
        score, sim = SimilarityMatcher._similarity(char_diff, -1.0, a, b)

        print score
        print sim

        self.assertEqual(12, score)

    def test_most_similar(self):

        word_dict = {}
        load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

        self.assertEquals(word_dict["I"], 1)

        sim = SimilarityMatcher()
        sim.update_from_dict(word_dict)

        score, word = sim.get_most_similar_word("iPod")

        self.assertEqual(word, "ipod")

    def test_expand_dictionary(self):

        word_dict = {}
        load_pos_tagged_data("Data/TweeboOct27.conll", worddict=word_dict)

        test_dict = {}

        load_pos_tagged_data("Data/TweeboDaily547.conll", worddict=test_dict)