<commit_before>from __future__ import (division, absolute_import,
                        print_function, unicode_literals)
import argparse
import logging
import sys
import os
import glob
import json
import pickle
import heapq
import numpy as np
from random import sample
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from optimisers import adam, adadelta, rmsprop, sgd
from embeddings import gloveDocumentParser
from sklearn.cross_validation import train_test_split

EPSILON_FOR_LOG = 1e-8

def get_encoder(context_encoder):
    def baseline_encoder(x, y, x_mask, y_pos, params, tparams):
        ''' baseline context encoder given one piece of text

        Returns ctx each row for a training instance
        '''
        if x.ndim == 1:
            mb_size = 1
        elif x.ndim == 2:
            mb_size = x.shape[0]
        seq_len = params['seq_maxlen']
        wv_size = params['full_text_word_vector_size']
        
        x_emb = tparams['Xemb'][x.flatten(), :]
        x_emb_masked = T.batched_dot(x_emb, x_mask.flatten())

        if x.ndim == 1:
            ctx = x_emb_masked.sum(axis=0) / x_mask.sum()
        elif x.ndim == 2:
            ctx = T.batched_dot(
                x_emb_masked.reshape((mb_size, seq_len, wv_size)).sum(axis=1),
                1 / x_mask.sum(axis=1)
            )
         
        return T.cast(ctx, theano.config.floatX) 

    def attention_encoder(x, y, x_mask, y_pos, params, tparams):
        ''' attention-based context encoder given one piece of text

        '''
        if x.ndim == 1:
            mb_size = 1
        else:
            mb_size = params['minibatch_size']
        l = params['seq_maxlen']
        C = params['summary_context_length']
        Q = params['attention_weight_max_roll']
        wv_size_x = params['full_text_word_vector_size']
        wv_size_y = params['summary_word_vector_size']
        P = tparams['att_P']
        m = tparams['att_P_conv']

        if x.ndim == 1:
            x_emb = tparams['Xemb'][x, :]
            y_emb = tparams['Yemb'][y[(y_pos - C):y_pos], :]
            p = T.nnet.softmax(
                T.dot(x_emb, T.dot(P, y_emb.flatten()))
            ).flatten()
            p_masked = p * x_mask
            p_masked /= p_masked.sum()
            ctx = T.batched_dot(x_emb, T.dot(m, p_masked)).sum(axis=0)

        elif x.ndim == 2:
            x_emb = tparams['Xemb'][x.flatten(), :]
            x_emb = x_emb.reshape((mb_size, l, wv_size_x))
            y_emb = tparams['Yemb'][y[:, (y_pos - C):y_pos].flatten(), :]
            y_emb = y_emb.flatten().reshape((mb_size, C * wv_size_y)).T
            p = T.nnet.softmax(
                    T.batched_dot(x_emb, T.dot(P, y_emb).T)
                    )
            p_masked = p * x_mask
            inverse_row_sum_ = (1 / p_masked.sum(axis=1)).flatten()
            p_masked = T.batched_dot(p_masked, inverse_row_sum_)
            ctx = T.batched_dot(T.dot(m, p_masked.T).T, x_emb) 

        return T.cast(ctx, theano.config.floatX)

    if context_encoder == 'baseline':
        return baseline_encoder
    elif context_encoder == 'attention':
        return  attention_encoder
    else:
        raise ValueError('Invalide context encoder {:}'.format(context_encoder))

def dropout_layer(state_before, params, tparams):
    trng = params['trng']
    use_noise = (params['phase'] == 'training')
    proj = tensor.switch(
        use_noise,
        state_before * trng.binomial(state_before.shape, 
                                     p=params['dropout_rate'], n=1,
                                     dtype=state_before.dtype),
        state_before * params['dropout_rate']
        )
    return proj
<commit_msg>refactor the attention-based context encoder<commit_after>from __future__ import (division, absolute_import,
                        print_function, unicode_literals)
import argparse
import logging
import sys
import os
import glob
import json
import pickle
import heapq
import numpy as np
from random import sample
import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from optimisers import adam, adadelta, rmsprop, sgd
from embeddings import gloveDocumentParser
from sklearn.cross_validation import train_test_split

EPSILON_FOR_LOG = 1e-8

def get_encoder(context_encoder):
    def baseline_encoder(x, y, x_mask, y_pos, params, tparams):
        ''' baseline context encoder given one piece of text

        Returns ctx each row for a training instance
        '''
        if x.ndim == 1:
            mb_size = 1
        elif x.ndim == 2:
            mb_size = x.shape[0]
        seq_len = params['seq_maxlen']
        wv_size = params['full_text_word_vector_size']
        
        x_emb = tparams['Xemb'][x.flatten(), :]
        x_emb_masked = T.batched_dot(x_emb, x_mask.flatten())

        if x.ndim == 1:
            ctx = x_emb_masked.sum(axis=0) / x_mask.sum()
        elif x.ndim == 2:
            ctx = T.batched_dot(
                x_emb_masked.reshape((mb_size, seq_len, wv_size)).sum(axis=1),
                1 / x_mask.sum(axis=1)
            )
         
        return T.cast(ctx, theano.config.floatX) 

    def attention_encoder(x, y, x_mask, y_pos, params, tparams):
        ''' attention-based context encoder given one piece of text

        '''
        if x.ndim == 1:
            mb_size = 1
        else:
            mb_size = params['minibatch_size']
        l = params['seq_maxlen']
        C = params['summary_context_length']
        Q = params['attention_weight_max_roll']
        wv_size_x = params['full_text_word_vector_size']
        wv_size_y = params['summary_word_vector_size']
        P = tparams['att_P']
        m = tparams['att_P_conv']

        if x.ndim == 1:
            x_emb = tparams['Xemb'][x, :]
            y_emb = tparams['Yemb'][y[(y_pos - C):y_pos], :]
            p = T.nnet.softmax(
                T.dot(x_emb, T.dot(P, y_emb.flatten()))
            ).flatten()
            p_masked = p * x_mask
            p_masked_n = p_masked / p_masked.sum()
            ctx = T.batched_dot(x_emb, T.dot(m, p_masked_n)).sum(axis=0)

        elif x.ndim == 2:
            x_emb = tparams['Xemb'][x.flatten(), :]
            x_emb = x_emb.reshape((mb_size, l, wv_size_x))
            y_emb = tparams['Yemb'][y[:, (y_pos - C):y_pos].flatten(), :]
            y_emb = y_emb.flatten().reshape((mb_size, C * wv_size_y)).T
            p = T.nnet.softmax(
                    T.batched_dot(x_emb, T.dot(P, y_emb).T)
                    )
            p_masked = p * x_mask
            p_masked_n = p_masked / p_masked.norm(1, axis=1).reshape((mb_size, 1))
            ctx = T.batched_dot(T.dot(m, p_masked_n.T).T, x_emb) 

        return T.cast(ctx, theano.config.floatX)

    if context_encoder == 'baseline':
        return baseline_encoder
    elif context_encoder == 'attention':
        return  attention_encoder
    else:
        raise ValueError('Invalide context encoder {:}'.format(context_encoder))

def dropout_layer(state_before, params, tparams):
    trng = params['trng']
    use_noise = (params['phase'] == 'training')
    proj = tensor.switch(
        use_noise,
        state_before * trng.binomial(state_before.shape, 
                                     p=params['dropout_rate'], n=1,
                                     dtype=state_before.dtype),
        state_before * params['dropout_rate']
        )
    return proj

{'fraction_non_alphanumeric': 0.1132733861226681, 'fraction_numerical': 0.005502617098376057, 'mean_word_length': 2.145631067961165, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3250709', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2462, 'n_words': 516}