<commit_before>import tensorflow as tf


# pylint: disable=too-few-public-methods
class OrthoGRUCell(tf.contrib.rnn.GRUCell):
    """Classic GRU cell but initialized using random orthogonal matrices."""

    def __init__(self,
                 num_units,
                 activation=None,
                 reuse=None,
                 bias_initializer=None):
        tf.contrib.rnn.GRUCell.__init__(
            self, num_units, activation, reuse, tf.orthogonal_initializer(),
            bias_initializer)

    def __call__(self, inputs, state, scope="OrthoGRUCell"):
        return tf.contrib.rnn.GRUCell.__call__(self, inputs, state, scope)


# Note that tensorflow does not like when the type annotations are present.
class NematusGRUCell(OrthoGRUCell):
    """Nematus implementation of gated recurrent unit cell.

    The main difference is the order in which the gating functions and linear
    projections are applied to the hidden state.

    The math is equivalent, in practice there are differences due to float
    precision errors.
    """
    def __init__(self, rnn_size, use_state_bias=False, use_input_bias=True):
        self.use_state_bias = use_state_bias
        self.use_input_bias = use_input_bias

        OrthoGRUCell.__init__(self, rnn_size)

    def call(self, inputs, state):
        """Gated recurrent unit (GRU) with nunits cells."""
        with tf.variable_scope("gates"):
            input_to_gates = tf.layers.dense(
                inputs, 2 * self._num_units, name="input_proj",
                use_bias=self.use_input_bias)

            # Nematus does the orthogonal initialization probably differently
            state_to_gates = tf.layers.dense(
                state, 2 * self._num_units,
                use_bias=self.use_state_bias,
                kernel_initializer=tf.orthogonal_initializer(),
                name="state_proj")

            gates_input = state_to_gates + input_to_gates
            reset, update = tf.split(
                tf.sigmoid(gates_input), num_or_size_splits=2, axis=1)

        with tf.variable_scope("candidate"):
            input_to_candidate = tf.layers.dense(
                inputs, self._num_units, use_bias=self.use_input_bias,
                name="input_proj")

            state_to_candidate = tf.layers.dense(
                state, self._num_units, use_bias=self.use_state_bias,
                name="state_proj")

            candidate = self._activation(
                state_to_candidate * reset + input_to_candidate)

        new_state = update * state + (1 - update) * candidate
        return new_state, new_state
<commit_msg>nematus cell is child of grucell not orthogrucell<commit_after>import tensorflow as tf


# pylint: disable=too-few-public-methods
class OrthoGRUCell(tf.contrib.rnn.GRUCell):
    """Classic GRU cell but initialized using random orthogonal matrices."""

    def __init__(self,
                 num_units,
                 activation=None,
                 reuse=None,
                 bias_initializer=None):
        tf.contrib.rnn.GRUCell.__init__(
            self, num_units, activation, reuse, tf.orthogonal_initializer(),
            bias_initializer)

    def __call__(self, inputs, state, scope="OrthoGRUCell"):
        return tf.contrib.rnn.GRUCell.__call__(self, inputs, state, scope)


# Note that tensorflow does not like when the type annotations are present.
class NematusGRUCell(tf.contrib.rnn.GRUCell):
    """Nematus implementation of gated recurrent unit cell.

    The main difference is the order in which the gating functions and linear
    projections are applied to the hidden state.

    The math is equivalent, in practice there are differences due to float
    precision errors.
    """

    def __init__(self, rnn_size, use_state_bias=False, use_input_bias=True):
        self.use_state_bias = use_state_bias
        self.use_input_bias = use_input_bias

        tf.contrib.rnn.GRUCell.__init__(self, rnn_size)

    def call(self, inputs, state):
        """Gated recurrent unit (GRU) with nunits cells."""
        with tf.variable_scope("gates"):
            input_to_gates = tf.layers.dense(
                inputs, 2 * self._num_units, name="input_proj",
                use_bias=self.use_input_bias)

            # Nematus does the orthogonal initialization probably differently
            state_to_gates = tf.layers.dense(
                state, 2 * self._num_units,
                use_bias=self.use_state_bias,
                kernel_initializer=tf.orthogonal_initializer(),
                name="state_proj")

            gates_input = state_to_gates + input_to_gates
            reset, update = tf.split(
                tf.sigmoid(gates_input), num_or_size_splits=2, axis=1)

        with tf.variable_scope("candidate"):
            input_to_candidate = tf.layers.dense(
                inputs, self._num_units, use_bias=self.use_input_bias,
                name="input_proj")

            state_to_candidate = tf.layers.dense(
                state, self._num_units, use_bias=self.use_state_bias,
                name="state_proj")

            candidate = self._activation(
                state_to_candidate * reset + input_to_candidate)

        new_state = update * state + (1 - update) * candidate
        return new_state, new_state

{'fraction_non_alphanumeric': 0.08161723030417532, 'fraction_numerical': 0.001889287738522577, 'mean_word_length': 2.4132817537072855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '344036', 'n_tokens_mistral': 1583, 'n_tokens_neox': 1494, 'n_words': 363}