<commit_before>import tensorflow as tf

def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      embedding_size, cell, output_size=None,
                      loop_function=None, dtype=tf.float32, scope=None):

    if output_size is None:
        output_size = cell.output_size

    with tf.variable_scope(scope or "attention_decoder"):
        batch_size = tf.shape(decoder_inputs[0])[0]    # Needed for reshaping.

        # do manualy broadcasting of the initial state if we want it
        # to be the same for all inputs
        if len(initial_state.get_shape()) == 1:
            state_size = initial_state.get_shape()[0].value
            initial_state = tf.reshape(tf.tile(initial_state,
                                               tf.shape(decoder_inputs[0])[:1]),
                                       [-1, state_size])

        state = initial_state
        outputs = []
        prev = None

        attns = [a.initialize(batch_size, dtype) for a in attention_objects]

        states = []
        for i, inp in enumerate(decoder_inputs):
            if i > 0:
                tf.get_variable_scope().reuse_variables()
            # If loop_function is set, we use it instead of decoder_inputs.
            if loop_function is not None and prev is not None:
                with tf.variable_scope("loop_function", reuse=True):
                    inp = loop_function(prev, i)
            # Merge input and previous attentions into one vector of the right
            # size.
            x = tf.nn.seq2seq.linear([inp] + attns, embedding_size, True)
            # Run the RNN.
            cell_output, state = cell(x, state)
            states.append(state)
            # Run the attention mechanism.
            attns = [a.attention(state) for a in attention_objects]

            if attns:
                with tf.variable_scope("AttnOutputProjection"):
                    output = tf.nn.seq2seq.linear([cell_output] + attns, output_size,
                                             True)
            else:
                output = cell_output

            if loop_function is not None:
                prev = output
            outputs.append(output)

    return outputs, states

class Attention(object):
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=None):
        self.scope = scope
        self.attentions_in_time = []
        self.attention_states = tf.nn.dropout(attention_states,
                                              dropout_placeholder)
        self.input_weights = input_weights

        with tf.variable_scope(scope):<commit_msg>[ci skip] added docstring to attention inint<commit_after>import tensorflow as tf

def attention_decoder(decoder_inputs, initial_state, attention_objects,
                      embedding_size, cell, output_size=None,
                      loop_function=None, dtype=tf.float32, scope=None):

    if output_size is None:
        output_size = cell.output_size

    with tf.variable_scope(scope or "attention_decoder"):
        batch_size = tf.shape(decoder_inputs[0])[0]    # Needed for reshaping.

        # do manualy broadcasting of the initial state if we want it
        # to be the same for all inputs
        if len(initial_state.get_shape()) == 1:
            state_size = initial_state.get_shape()[0].value
            initial_state = tf.reshape(tf.tile(initial_state,
                                               tf.shape(decoder_inputs[0])[:1]),
                                       [-1, state_size])

        state = initial_state
        outputs = []
        prev = None

        attns = [a.initialize(batch_size, dtype) for a in attention_objects]

        states = []
        for i, inp in enumerate(decoder_inputs):
            if i > 0:
                tf.get_variable_scope().reuse_variables()
            # If loop_function is set, we use it instead of decoder_inputs.
            if loop_function is not None and prev is not None:
                with tf.variable_scope("loop_function", reuse=True):
                    inp = loop_function(prev, i)
            # Merge input and previous attentions into one vector of the right
            # size.
            x = tf.nn.seq2seq.linear([inp] + attns, embedding_size, True)
            # Run the RNN.
            cell_output, state = cell(x, state)
            states.append(state)
            # Run the attention mechanism.
            attns = [a.attention(state) for a in attention_objects]

            if attns:
                with tf.variable_scope("AttnOutputProjection"):
                    output = tf.nn.seq2seq.linear([cell_output] + attns, output_size,
                                             True)
            else:
                output = cell_output

            if loop_function is not None:
                prev = output
            outputs.append(output)

    return outputs, states

class Attention(object):
    def __init__(self, attention_states, scope, dropout_placeholder,
                 input_weights=None, max_fertility=None):
        """Create the attention object.

        Args:
            attention_states: A Tensor of shape (batch x time x state_size)
                              with the output states of the encoder.
            scope: The name of the variable scope in the graph used by this
                   attention object.
            dropout_placeholder: A Tensor that contains the value of the dropout
                                 keep probability
            input_weights: (Optional) The padding weights on the input.
            max_fertility: (Optional) For the Coverage attention compatibilty,
                           maximum fertility of one word.
        """
        self.scope = scope
        self.attentions_in_time = []
        self.attention_states = tf.nn.dropout(attention_states,
                                              dropout_placeholder)
        self.input_weights = input_weights

        with tf.variable_scope(scope):
{'fraction_non_alphanumeric': 0.07338842975206611, 'fraction_numerical': 0.003966942148760331, 'mean_word_length': 1.720773381294964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3202337', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1503, 'n_words': 438}