<commit_before>import sys
import tensorflow as tf

# ModuleNotFoundError is new in 3.6; older versions will throw SystemError
if sys.version_info < (3, 6):
    ModuleNotFoundError = SystemError

try:
    from . import layers
    from . import model_inputs
    from .sampling_utils import SamplingUtils
except (ModuleNotFoundError, ImportError) as e:
    import layers
    import model_inputs
    from sampling_utils import SamplingUtils


"""Builds a GRU-based attentional encoder-decoder model.

This class is responsible for constructing a TensorFlow graph that takes
a minibatch of sentence pairs as input and calculates a loss value via
teacher forcing (specifically, the loss is the mean sentence-level
cross-entropy).

For inference (sampling and beam search), see rnn_inference.py.

For optimization, see model_updater.py.
"""
class RNNModel(object):
    def __init__(self, config):
        self.inputs = model_inputs.ModelInputs(config)

        # Dropout functions for words.
        # These probabilistically zero-out all embedding values for individual
        # words.
        dropout_source, dropout_target = None, None
        if config.rnn_use_dropout and config.rnn_dropout_source > 0.0:
            def dropout_source(x):
                return tf.compat.v1.layers.dropout(
                    x, noise_shape=(tf.shape(input=x)[0], tf.shape(input=x)[1], 1),
                    rate=config.rnn_dropout_source,
                    training=self.inputs.training)
        if config.rnn_use_dropout and config.rnn_dropout_target > 0.0:
            def dropout_target(y):
                return tf.compat.v1.layers.dropout(
                    y, noise_shape=(tf.shape(input=y)[0], tf.shape(input=y)[1], 1),
                    rate=config.rnn_dropout_target,
                    training=self.inputs.training)

        # Dropout functions for use within FF, GRU, and attention layers.
        # We use Gal and Ghahramani (2016)-style dropout, so these functions
        # will be used to create 2D dropout masks that are reused at every
        # timestep.
        dropout_embedding, dropout_hidden = None, None
        if config.rnn_use_dropout and config.rnn_dropout_embedding > 0.0:
            def dropout_embedding(e):
                return tf.compat.v1.layers.dropout(e, noise_shape=tf.shape(input=e),
                                         rate=config.rnn_dropout_embedding,
                                         training=self.inputs.training)
        if config.rnn_use_dropout and config.rnn_dropout_hidden > 0.0:
            def dropout_hidden(h):
                return tf.compat.v1.layers.dropout(h, noise_shape=tf.shape(input=h),
                                         rate=config.rnn_dropout_hidden,
                                         training=self.inputs.training)

        batch_size = tf.shape(input=self.inputs.x)[-1]  # dynamic value

        with tf.compat.v1.variable_scope("encoder"):
            self.encoder = Encoder(config, batch_size, dropout_source,
                                   dropout_embedding, dropout_hidden)
            ctx, embs = self.encoder.get_context(self.inputs.x, self.inputs.x_mask)

        with tf.compat.v1.variable_scope("decoder"):
            if config.tie_encoder_decoder_embeddings:
                tied_embeddings = self.encoder.emb_layer
            else:
                tied_embeddings = None
            self.decoder = Decoder(config, ctx, embs, self.inputs.x_mask,
                                   dropout_target, dropout_embedding,
                                   dropout_hidden, tied_embeddings)
            self.logits = self.decoder.score(self.inputs.y)

        with tf.compat.v1.variable_scope("loss"):
            self.loss_layer = layers.Masked_cross_entropy_loss(
                self.inputs.y, self.inputs.y_mask, config.label_smoothing,
                training=self.inputs.training)
            self._loss_per_sentence = self.loss_layer.forward(self.logits)
            self._loss = tf.reduce_mean(input_tensor=self._loss_per_sentence, keepdims=False)
<commit_msg>fix rnn_dropout_embedding with TF >= 1.15<commit_after>import sys
import tensorflow as tf

# ModuleNotFoundError is new in 3.6; older versions will throw SystemError
if sys.version_info < (3, 6):
    ModuleNotFoundError = SystemError

try:
    from . import layers
    from . import model_inputs
    from .sampling_utils import SamplingUtils
except (ModuleNotFoundError, ImportError) as e:
    import layers
    import model_inputs
    from sampling_utils import SamplingUtils


"""Builds a GRU-based attentional encoder-decoder model.

This class is responsible for constructing a TensorFlow graph that takes
a minibatch of sentence pairs as input and calculates a loss value via
teacher forcing (specifically, the loss is the mean sentence-level
cross-entropy).

For inference (sampling and beam search), see rnn_inference.py.

For optimization, see model_updater.py.
"""
class RNNModel(object):
    def __init__(self, config):
        self.inputs = model_inputs.ModelInputs(config)

        # Dropout functions for words.
        # These probabilistically zero-out all embedding values for individual
        # words.
        dropout_source, dropout_target = None, None
        if config.rnn_use_dropout and config.rnn_dropout_source > 0.0:
            def dropout_source(x):
                return tf.compat.v1.layers.dropout(
                    x, noise_shape=(tf.shape(input=x)[0], tf.shape(input=x)[1], 1),
                    rate=config.rnn_dropout_source,
                    training=self.inputs.training)
        if config.rnn_use_dropout and config.rnn_dropout_target > 0.0:
            def dropout_target(y):
                return tf.compat.v1.layers.dropout(
                    y, noise_shape=(tf.shape(input=y)[0], tf.shape(input=y)[1], 1),
                    rate=config.rnn_dropout_target,
                    training=self.inputs.training)

        # Dropout functions for use within FF, GRU, and attention layers.
        # We use Gal and Ghahramani (2016)-style dropout, so these functions
        # will be used to create 2D dropout masks that are reused at every
        # timestep.
        dropout_embedding, dropout_hidden = None, None
        if config.rnn_use_dropout and config.rnn_dropout_embedding > 0.0:
            def dropout_embedding(e):
                return tf.compat.v1.layers.dropout(e, noise_shape=(tf.shape(input=e)[0], tf.shape(input=e)[1]),
                                         rate=config.rnn_dropout_embedding,
                                         training=self.inputs.training)
        if config.rnn_use_dropout and config.rnn_dropout_hidden > 0.0:
            def dropout_hidden(h):
                return tf.compat.v1.layers.dropout(h, noise_shape=(tf.shape(input=h)[0], tf.shape(input=h)[1]),
                                         rate=config.rnn_dropout_hidden,
                                         training=self.inputs.training)

        batch_size = tf.shape(input=self.inputs.x)[-1]  # dynamic value

        with tf.compat.v1.variable_scope("encoder"):
            self.encoder = Encoder(config, batch_size, dropout_source,
                                   dropout_embedding, dropout_hidden)
            ctx, embs = self.encoder.get_context(self.inputs.x, self.inputs.x_mask)

        with tf.compat.v1.variable_scope("decoder"):
            if config.tie_encoder_decoder_embeddings:
                tied_embeddings = self.encoder.emb_layer
            else:
                tied_embeddings = None
            self.decoder = Decoder(config, ctx, embs, self.inputs.x_mask,
                                   dropout_target, dropout_embedding,
                                   dropout_hidden, tied_embeddings)
            self.logits = self.decoder.score(self.inputs.y)

        with tf.compat.v1.variable_scope("loss"):
            self.loss_layer = layers.Masked_cross_entropy_loss(
                self.inputs.y, self.inputs.y_mask, config.label_smoothing,
                training=self.inputs.training)
            self._loss_per_sentence = self.loss_layer.forward(self.logits)
            self._loss = tf.reduce_mean(input_tensor=self._loss_per_sentence, keepdims=False)

{'fraction_non_alphanumeric': 0.08234006324495256, 'fraction_numerical': 0.00839211870591097, 'mean_word_length': 2.215877982010168, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3583982', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2294, 'n_words': 532}