<commit_before>import tensorflow as tf

from neuralmonkey.logging import debug

def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      cell, maxout_size, loop_function=None,
                      dtype=tf.float32, scope=None):
    outputs = []
    states = []

    debug("initial state shape: {}".format(initial_state.get_shape()))

    #### WTF does this do?
    # do manualy broadcasting of the initial state if we want it
    # to be the same for all inputs
    if len(initial_state.get_shape()) == 1:
        debug("Warning! I am doing this weird job.")
        state_size = initial_state.get_shape()[0].value
        initial_state = tf.reshape(tf.tile(initial_state,
                                           tf.shape(decoder_inputs[0])[:1]),
                                   [-1, state_size])

    with tf.variable_scope(scope or "attention_decoder"):
        output, state = decode_step(decoder_inputs[0], initial_state,
                                    attention_objects, cell, maxout_size)
        outputs.append(output)
        states.append(state)

        for step in range(1, len(decoder_inputs)):
            tf.get_variable_scope().reuse_variables()

            debug("State shape: {}".format(state.get_shape()))

            if loop_function:
                decoder_input = loop_function(output, step)
            else:
                decoder_input = decoder_inputs[step]

            output, state = decode_step(decoder_input, state, attention_objects,
                                        cell, maxout_size)
            outputs.append(output)
            states.append(state)

    return outputs, states


def decode_step(prev_output, prev_state, attention_objects,
                rnn_cell, maxout_size):
    """This function implements equations in section A.2.2 of the
    Bahdanau et al. (2015) paper, on pages 13 and 14.

    Arguments:
        prev_output: Previous decoded output (denoted by y_i-1)
        prev_state: Previous state (denoted by s_i-1)
        attention_objects: Objects that do attention
        rnn_cell: The RNN cell to use (should be GRU)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        Tuple of the new output and state
    """
    ## compute c_i:
    contexts = [a.attention(prev_state) for a in attention_objects]

    ## compute t_i:
    output = maxout_projection(prev_state, prev_output, contexts, maxout_size)

    ## compute s_i based on y_i-1, c_i and s_i-1
    _, state = rnn_cell(tf.concat(1, [prev_output] + contexts), prev_state)

    return output, state


def linear_projection(states, size):
    """Adds linear projection on top of a list of vectors.

    Arguments:
        states: A list of states to project
        size: Size of the resulting vector

    Returns the projected vector of the specified size.
    """
    with tf.variable_scope("AttnOutputProjection"):
        output = tf.nn.seq2seq.linear(states, size, True)
        return output


def maxout_projection(prev_state, prev_output, current_contexts, maxout_size):
    """ Adds maxout hidden layer for computation the output
    distribution.

    In Bahdanau: t_i = [max(t'_{i,2j-1}, t'_{i,2j})] ... j = 1,...,l
    where: t' is linear projection of concatenation of the previous
    state, previous output and current attention contexts.

    Arguments:
        prev_state: Previous state (s_i-1 in the paper)
        prev_output: Embedding of previously outputted word (y_i-1)
        current_contexts: List of attention vectors (only one in the paper,
                          denoted by c_i)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        A tensor of shape batch x maxout_size
    """
    with tf.variable_scope("AttnMaxoutProjection"):
        input_ = [prev_state, prev_output] + current_contexts
        projected = tf.nn.seq2seq.linear(input_, maxout_size * 2, True)

        ## dropouts?!

        # projected je batch x (2*maxout_size)
        # potreba narezat po dvojicich v dimenzi jedna
        # aby byl batch x 2 x maxout_size

        maxout_input = tf.reshape(projected, [-1, 1, 2, maxout_size])

        maxpooled = tf.nn.max_pool(
            maxout_input, [1, 1, 2, 1], [1, 1, 2, 1], "SAME")

        reshaped = tf.reshape(maxpooled, [-1, maxout_size])
        return reshaped


class Attention(object):<commit_msg>[ci skip] add goodfellow citation to maxout<commit_after>import tensorflow as tf

from neuralmonkey.logging import debug

def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      cell, maxout_size, loop_function=None,
                      dtype=tf.float32, scope=None):
    outputs = []
    states = []

    debug("initial state shape: {}".format(initial_state.get_shape()))

    #### WTF does this do?
    # do manualy broadcasting of the initial state if we want it
    # to be the same for all inputs
    if len(initial_state.get_shape()) == 1:
        debug("Warning! I am doing this weird job.")
        state_size = initial_state.get_shape()[0].value
        initial_state = tf.reshape(tf.tile(initial_state,
                                           tf.shape(decoder_inputs[0])[:1]),
                                   [-1, state_size])

    with tf.variable_scope(scope or "attention_decoder"):
        output, state = decode_step(decoder_inputs[0], initial_state,
                                    attention_objects, cell, maxout_size)
        outputs.append(output)
        states.append(state)

        for step in range(1, len(decoder_inputs)):
            tf.get_variable_scope().reuse_variables()

            debug("State shape: {}".format(state.get_shape()))

            if loop_function:
                decoder_input = loop_function(output, step)
            else:
                decoder_input = decoder_inputs[step]

            output, state = decode_step(decoder_input, state, attention_objects,
                                        cell, maxout_size)
            outputs.append(output)
            states.append(state)

    return outputs, states


def decode_step(prev_output, prev_state, attention_objects,
                rnn_cell, maxout_size):
    """This function implements equations in section A.2.2 of the
    Bahdanau et al. (2015) paper, on pages 13 and 14.

    Arguments:
        prev_output: Previous decoded output (denoted by y_i-1)
        prev_state: Previous state (denoted by s_i-1)
        attention_objects: Objects that do attention
        rnn_cell: The RNN cell to use (should be GRU)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        Tuple of the new output and state
    """
    ## compute c_i:
    contexts = [a.attention(prev_state) for a in attention_objects]

    ## compute t_i:
    output = maxout_projection(prev_state, prev_output, contexts, maxout_size)

    ## compute s_i based on y_i-1, c_i and s_i-1
    _, state = rnn_cell(tf.concat(1, [prev_output] + contexts), prev_state)

    return output, state


def linear_projection(states, size):
    """Adds linear projection on top of a list of vectors.

    Arguments:
        states: A list of states to project
        size: Size of the resulting vector

    Returns the projected vector of the specified size.
    """
    with tf.variable_scope("AttnOutputProjection"):
        output = tf.nn.seq2seq.linear(states, size, True)
        return output


def maxout_projection(prev_state, prev_output, current_contexts, maxout_size):
    """ Adds maxout hidden layer for computation the output
    distribution.

    In Bahdanau: t_i = [max(t'_{i,2j-1}, t'_{i,2j})] ... j = 1,...,l
    where: t' is linear projection of concatenation of the previous
    state, previous output and current attention contexts.

    More about maxout in Goodfellow et al. (2013)
    here: https://arxiv.org/pdf/1302.4389.pdf

    Arguments:
        prev_state: Previous state (s_i-1 in the paper)
        prev_output: Embedding of previously outputted word (y_i-1)
        current_contexts: List of attention vectors (only one in the paper,
                          denoted by c_i)
        maxout_size: The size of the maxout hidden layer (denoted by l)

    Returns:
        A tensor of shape batch x maxout_size
    """
    with tf.variable_scope("AttnMaxoutProjection"):
        input_ = [prev_state, prev_output] + current_contexts
        projected = tf.nn.seq2seq.linear(input_, maxout_size * 2, True)

        ## dropouts?!

        # projected je batch x (2*maxout_size)
        # potreba narezat po dvojicich v dimenzi jedna
        # aby byl batch x 2 x maxout_size

        maxout_input = tf.reshape(projected, [-1, 1, 2, maxout_size])

        maxpooled = tf.nn.max_pool(
            maxout_input, [1, 1, 2, 1], [1, 1, 2, 1], "SAME")

        reshaped = tf.reshape(maxpooled, [-1, maxout_size])
        return reshaped


class Attention(object):
{'fraction_non_alphanumeric': 0.08894848586434238, 'fraction_numerical': 0.011844898871382277, 'mean_word_length': 2.4317484662576687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '648712', 'n_tokens_mistral': 2742, 'n_tokens_neox': 2571, 'n_words': 799}