# Path: utils.py
# def init_embedding_table(filename='embeddings/vocab_embeddings.txt', embed_dim=300, vocab_file='squad_rare/vocab.txt'):
#     import theano
#     vocab = ['<DUMMY>', '<EOA>', '@placeholder', '<UNK>'] + [ w.strip().split()[0] for w in open(vocab_file) ]
#     reverse_vocab = {w: i for i, w in enumerate(vocab)}
#     word_vecs = read_word_vectors(filename,embed_dim)
#     embeddings = numpy.ndarray(shape=(len(vocab), embed_dim),dtype=theano.config.floatX)
#     count = 0
#     for k,v in word_vecs.iteritems():
#         if k.upper() in ['<DUMMY>', '<EOA>', '@placeholder', '<UNK>']:
#             k = k.upper()
#         # print (count)
#         # print (reverse_vocab[k])
#         count += 1
#         embeddings[reverse_vocab[k],:] = v
# 
#     return embeddings


import theano
import numpy
import scipy
from theano import tensor
from blocks.bricks import Tanh, Softmax, Linear, MLP, Identity, Rectifier
from blocks.bricks.lookup import LookupTable
from blocks.bricks.recurrent import LSTM, GatedRecurrent, Bidirectional
from blocks.bricks.attention import SequenceContentAttention
from blocks.bricks.parallel import Fork
from blocks.bricks.sequence_generators import (
    SequenceGenerator, Readout, SoftmaxEmitter, LookupFeedback)
from blocks.filter import VariableFilter
from blocks.roles import WEIGHT
from blocks.graph import ComputationGraph, apply_dropout, apply_noise
from blocks.monitoring import aggregation
from blocks.initialization import Orthogonal, IsotropicGaussian, Constant
from utils import init_embedding_table
        att_weights_question = att_weights_question.dimshuffle(0, 2, 1) # --> lc,bs,lq
        att_weights_question = att_weights_question.reshape((att_weights_question.shape[0]*att_weights_question.shape[1], att_weights_question.shape[2])) # --> lc*bs,lq
        att_weights_question = tensor.nnet.softmax(att_weights_question) # softmax over axis 1 -> length of question # --> lc*bs,lq
        att_weights_question = att_weights_question.reshape((c_hidden.shape[0], q_hidden.shape[1], q_hidden.shape[0])) # --> lc,bs,lq
        att_weights_question = att_weights_question.dimshuffle(0, 2, 1) # --> lc,lq,bs

        attended_question = tensor.sum(q_hidden[None, :, :, :] * att_weights_question[:, :, :, None], axis=1) # sum over axis 1 -> length of question --> lc,bs,dim
        attended_question.name = 'attended_question'

         # Match LSTM
        cqembed = tensor.concatenate([c_hidden, attended_question], axis=2)
        mlstms, mhidden_list = make_bidir_lstm_stack(cqembed, 2 * config.pre_lstm_size, context_mask.astype(theano.config.floatX),
                                                     config.match_lstm_size, config.match_skip_connections, 'match')
        bricks = bricks + mlstms
        if config.match_skip_connections:
            menc_dim = 2*sum(config.match_lstm_size)
            menc = tensor.concatenate(mhidden_list, axis=2)
        else:
            menc_dim = 2*config.match_lstm_size[-1]
            menc = tensor.concatenate(mhidden_list[-2:], axis=2)
        menc.name = 'menc'

        # Attention mechanism MLP start
        attention_mlp_start = MLP(dims=config.attention_mlp_hidden + [1],
                            activations=config.attention_mlp_activations[1:] + [Identity()],
                            name='attention_mlp_start')
        attention_clinear_start = Linear(input_dim=menc_dim, output_dim=config.attention_mlp_hidden[0], name='attm_start') # Wym
        bricks += [attention_mlp_start, attention_clinear_start]
        layer1_start = Tanh(name='layer1_start')
        layer1_start = layer1_start.apply(attention_clinear_start.apply(menc.reshape((menc.shape[0]*menc.shape[1], menc.shape[2])))

# Path: utils.py
# def init_embedding_table(filename='embeddings/vocab_embeddings.txt', embed_dim=300, vocab_file='squad_rare/vocab.txt'):
#     import theano
#     vocab = ['<DUMMY>', '<EOA>', '@placeholder', '<UNK>'] + [ w.strip().split()[0] for w in open(vocab_file) ]
#     reverse_vocab = {w: i for i, w in enumerate(vocab)}
#     word_vecs = read_word_vectors(filename,embed_dim)
#     embeddings = numpy.ndarray(shape=(len(vocab), embed_dim),dtype=theano.config.floatX)
#     count = 0
#     for k,v in word_vecs.iteritems():
#         if k.upper() in ['<DUMMY>', '<EOA>', '@placeholder', '<UNK>']:
#             k = k.upper()
#         # print (count)
#         # print (reverse_vocab[k])
#         count += 1
#         embeddings[reverse_vocab[k],:] = v
# 
#     return embeddings


# Path: model/match_lstm.py
import theano
import numpy
import scipy
from theano import tensor
from blocks.bricks import Tanh, Softmax, Linear, MLP, Identity, Rectifier
from blocks.bricks.lookup import LookupTable
from blocks.bricks.recurrent import LSTM, GatedRecurrent, Bidirectional
from blocks.bricks.attention import SequenceContentAttention
from blocks.bricks.parallel import Fork
from blocks.bricks.sequence_generators import (
    SequenceGenerator, Readout, SoftmaxEmitter, LookupFeedback)
from blocks.filter import VariableFilter
from blocks.roles import WEIGHT
from blocks.graph import ComputationGraph, apply_dropout, apply_noise
from blocks.monitoring import aggregation
from blocks.initialization import Orthogonal, IsotropicGaussian, Constant
from utils import init_embedding_table

        att_weights_question = att_weights_question.dimshuffle(0, 2, 1) # --> lc,bs,lq
        att_weights_question = att_weights_question.reshape((att_weights_question.shape[0]*att_weights_question.shape[1], att_weights_question.shape[2])) # --> lc*bs,lq
        att_weights_question = tensor.nnet.softmax(att_weights_question) # softmax over axis 1 -> length of question # --> lc*bs,lq
        att_weights_question = att_weights_question.reshape((c_hidden.shape[0], q_hidden.shape[1], q_hidden.shape[0])) # --> lc,bs,lq
        att_weights_question = att_weights_question.dimshuffle(0, 2, 1) # --> lc,lq,bs

        attended_question = tensor.sum(q_hidden[None, :, :, :] * att_weights_question[:, :, :, None], axis=1) # sum over axis 1 -> length of question --> lc,bs,dim
        attended_question.name = 'attended_question'

         # Match LSTM
        cqembed = tensor.concatenate([c_hidden, attended_question], axis=2)
        mlstms, mhidden_list = make_bidir_lstm_stack(cqembed, 2 * config.pre_lstm_size, context_mask.astype(theano.config.floatX),
                                                     config.match_lstm_size, config.match_skip_connections, 'match')
        bricks = bricks + mlstms
        if config.match_skip_connections:
            menc_dim = 2*sum(config.match_lstm_size)
            menc = tensor.concatenate(mhidden_list, axis=2)
        else:
            menc_dim = 2*config.match_lstm_size[-1]
            menc = tensor.concatenate(mhidden_list[-2:], axis=2)
        menc.name = 'menc'

        # Attention mechanism MLP start
        attention_mlp_start = MLP(dims=config.attention_mlp_hidden + [1],
                            activations=config.attention_mlp_activations[1:] + [Identity()],
                            name='attention_mlp_start')
        attention_clinear_start = Linear(input_dim=menc_dim, output_dim=config.attention_mlp_hidden[0], name='attm_start') # Wym
        bricks += [attention_mlp_start, attention_clinear_start]
        layer1_start = Tanh(name='layer1_start')
        layer1_start = layer1_start.apply(attention_clinear_start.apply(menc.reshape((menc.shape[0]*menc.shape[1], menc.shape[2])))

                                        .reshape((menc.shape[0],menc.shape[1],config.attention_mlp_hidden[0])))