# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels
# 
# def prepare_data(char_seqs, labels, maxw, maxwlen, dim_proj):
#     """
#     Create the matrices from the datasets.
# 
#     This pad each sequence to the same length: the length of the
#     longest sequence or maxlen.
# 
#     if maxlen is set, we will cut all sequences to this maximum
#     length
# 
#     This swap the axis!
#     """
# 
#     # x: a list of sentences
#     n_samples = len(char_seqs)
# 
#     x_c = numpy.zeros((maxw, maxwlen, n_samples)).astype('int8')
#     x_mask = numpy.zeros((maxw, maxwlen, n_samples, dim_proj)).astype(theano.config.floatX)
#     y = numpy.zeros((maxw, n_samples)).astype('int8')
#     y_mask = numpy.zeros((maxw, n_samples)).astype('int8')
# 
#     for idx, (s_c, l) in enumerate(zip(char_seqs, labels)):
#         # idx is the current position in the mini-batch
#         # s_c is a list of characters
#         # s_w is a list of words
#         # l is a list of labels
#         c = 0
#         p = 0
#         warning = None
#         for j, a in enumerate(s_c):
#             # j is the current character in this tweet
#             # idx is the current tweet in this minibatch
#             # c is the current word (can be up to 16)
#             # p is the current character in this word
# 
#             if a == 0:
#                 # This current character is a space
#                 # Increase the word count and continue
#                 c += 1
#                 p = 0
#                 j += 1 # Temporarily skip to next loop char
#                 if c >= maxw:
#                     if j != len(s_c):
#                         warning = "truncation: too many words in this tweet! {}-{}".format(j, len(s_c))
#                     break
#                 if c >= len(l):
#                     if j != len(s_c):
#                         warning = "truncation: too many words for these labels {}-{}".format(j, len(s_c))
#                     break
# 
#             if p >= x_c.shape[1]:
#                 warning = "truncation: too many characters for this maxwlen"
#             else:
#                 x_c[c, p, idx] = a
#                 x_mask[c, p, idx] = numpy.ones(dim_proj)
# 
#             y[c, idx] = l[c]
#             y_mask[c, idx] = 1
#             p += 1
# 
#         if warning is not None:
#             #logging.warning("%s", warning)
#             pass
# 
#     return x_c, x_mask, y, y_mask


import unittest
import logging
import numpy as np
from modelio import load_pos_tagged_data, prepare_data
        self.assertEquals(char['A'], 19)
        self.assertEquals(char['N'], 20)
        self.assertEquals(char['K'], 21)
        self.assertEquals(char['S'], 22)
        self.assertEquals(char['r'], 23)
        self.assertEquals(char['k'], 24)
        self.assertEquals(char['!'], 25)
        self.assertEquals(len(char), 25)

        self.assertEquals(pos['~'], 1)
        self.assertEquals(pos['@'], 2)
        self.assertEquals(pos['!'], 3)
        self.assertEquals(pos[','], 4)
        self.assertEquals(pos['N'], 5)
        self.assertEquals(len(pos), 5)

        self.assertEquals(chars[0][0:3], [1, 2, 0])
        self.assertEquals(chars[0][3:13], [3, 4, 4, 5, 6, 7, 8, 9, 6, 0])
        self.assertEquals(words[0][0:3], [1, 1, 1])
        self.assertEquals(words[0][3:13], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2])
        self.assertEquals(labels, [[1, 2, 1, 2, 3, 4, 3, 5, 5, 4]])

    def test_load_pos_tagged_data_2(self):
        """
            Check we can handle a sample of 10 correctly.
        """

        char = {}
        word = {}
        pos = {}

# Path: modelio.py
# def load_pos_tagged_data(path, chardict = {}, worddict={}, posdict={}, overlap=15, allow_append=True):
# 
#     if allow_append:
#         build_character_dictionary(path, chardict)
#         build_word_dictionary(path, worddict)
#         build_tag_dictionary(path, posdict)
# 
#     cur_chars, cur_words, cur_labels = [], [], []
#     words, chars, labels = [], [], []
#     with open(path, 'r') as fin:
#         for line in fin:
#             line = line.strip()
#             if len(line) == 0:
#                 chars.append(cur_chars[:-1])
#                 words.append(cur_words[:-1])
#                 labels.append(cur_labels)
#                 cur_chars = []
#                 cur_labels = []
#                 cur_words = []
#                 continue
# 
#             word, pos = line.split('\t')
# 
#             if word not in worddict and allow_append:
#                 worddict[word] = len(worddict)+1
# 
#             for c in word:
#                 if c not in chardict and allow_append:
#                     chardict[c] = len(chardict)+1
# 
#                 if c in chardict:
#                     cur_chars.append(chardict[c])
#                 else:
#                     cur_chars.append(0)
# 
#                 if word in worddict:
#                     cur_words.append(worddict[word])
#                 else:
#                     cur_words.append(0)
# 
#                 if pos not in posdict and allow_append:
#                     posdict[pos] = len(posdict)+1
# 
#             if pos in posdict:
#                 cur_labels.append(posdict[pos])
#             else:
#                 cur_labels.append(0)
# 
#             if word in worddict:
#                 cur_words.append(worddict[word])
#             else:
#                 cur_words.append(0)
#             cur_chars.append(0)
# 
#     if len(cur_chars) > 0:
#     	chars.append(cur_chars)
#         words.append(cur_words)
#     	labels.append(cur_labels)
# 
#     return chars, words, labels
# 
# def prepare_data(char_seqs, labels, maxw, maxwlen, dim_proj):
#     """
#     Create the matrices from the datasets.
# 
#     This pad each sequence to the same length: the length of the
#     longest sequence or maxlen.
# 
#     if maxlen is set, we will cut all sequences to this maximum
#     length
# 
#     This swap the axis!
#     """
# 
#     # x: a list of sentences
#     n_samples = len(char_seqs)
# 
#     x_c = numpy.zeros((maxw, maxwlen, n_samples)).astype('int8')
#     x_mask = numpy.zeros((maxw, maxwlen, n_samples, dim_proj)).astype(theano.config.floatX)
#     y = numpy.zeros((maxw, n_samples)).astype('int8')
#     y_mask = numpy.zeros((maxw, n_samples)).astype('int8')
# 
#     for idx, (s_c, l) in enumerate(zip(char_seqs, labels)):
#         # idx is the current position in the mini-batch
#         # s_c is a list of characters
#         # s_w is a list of words
#         # l is a list of labels
#         c = 0
#         p = 0
#         warning = None
#         for j, a in enumerate(s_c):
#             # j is the current character in this tweet
#             # idx is the current tweet in this minibatch
#             # c is the current word (can be up to 16)
#             # p is the current character in this word
# 
#             if a == 0:
#                 # This current character is a space
#                 # Increase the word count and continue
#                 c += 1
#                 p = 0
#                 j += 1 # Temporarily skip to next loop char
#                 if c >= maxw:
#                     if j != len(s_c):
#                         warning = "truncation: too many words in this tweet! {}-{}".format(j, len(s_c))
#                     break
#                 if c >= len(l):
#                     if j != len(s_c):
#                         warning = "truncation: too many words for these labels {}-{}".format(j, len(s_c))
#                     break
# 
#             if p >= x_c.shape[1]:
#                 warning = "truncation: too many characters for this maxwlen"
#             else:
#                 x_c[c, p, idx] = a
#                 x_mask[c, p, idx] = numpy.ones(dim_proj)
# 
#             y[c, idx] = l[c]
#             y_mask[c, idx] = 1
#             p += 1
# 
#         if warning is not None:
#             #logging.warning("%s", warning)
#             pass
# 
#     return x_c, x_mask, y, y_mask


# Path: test_io.py
import unittest
import logging
import numpy as np
from modelio import load_pos_tagged_data, prepare_data

        self.assertEquals(char['A'], 19)
        self.assertEquals(char['N'], 20)
        self.assertEquals(char['K'], 21)
        self.assertEquals(char['S'], 22)
        self.assertEquals(char['r'], 23)
        self.assertEquals(char['k'], 24)
        self.assertEquals(char['!'], 25)
        self.assertEquals(len(char), 25)

        self.assertEquals(pos['~'], 1)
        self.assertEquals(pos['@'], 2)
        self.assertEquals(pos['!'], 3)
        self.assertEquals(pos[','], 4)
        self.assertEquals(pos['N'], 5)
        self.assertEquals(len(pos), 5)

        self.assertEquals(chars[0][0:3], [1, 2, 0])
        self.assertEquals(chars[0][3:13], [3, 4, 4, 5, 6, 7, 8, 9, 6, 0])
        self.assertEquals(words[0][0:3], [1, 1, 1])
        self.assertEquals(words[0][3:13], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2])
        self.assertEquals(labels, [[1, 2, 1, 2, 3, 4, 3, 5, 5, 4]])

    def test_load_pos_tagged_data_2(self):
        """
            Check we can handle a sample of 10 correctly.
        """

        char = {}
        word = {}
        pos = {}

        chars, words, labels = load_pos_tagged_data("Data/test_read_2.conll", char, word, pos)