<commit_before>""" TensorFlow Layers

Convenience functions but Input and Output should be tensors.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from contextlib import contextmanager
from functools import partial, wraps

import tensorflow as tf
from tensorflow.contrib import seq2seq


_phase = tf.Variable(False, name='phase', trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES])
_phase_train = _phase.assign(True)
_phase_infer = _phase.assign(False)


# TODO: move to ops
def _rank(x):
    return len(x.get_shape())


def _apply_dropout_mask(tensor_shape, keep_prob=1.0, normalize=True):
    random_tensor = keep_prob + tf.random_uniform(tensor_shape, dtype=tf.float32)
    binary_mask = tf.floor(random_tensor)
    if normalize:
        binary_mask = tf.reciprocal(keep_prob) * binary_mask
    return binary_mask


def _global_keep_prob(keep_prob):
    keep_prob = tf.convert_to_tensor(keep_prob, dtype=tf.float32)
    keep_prob = tf.cond(_phase, lambda: keep_prob, lambda: keep_prob * 0.0 + 1.0)
    return keep_prob


# TODO: add automatic naming
def pipe(func):

    class Pipe(object):
        def __init__(self, *args, **kwargs):
            self.func = func
            self.args = args
            self.kwargs = kwargs
            self.name = self.kwargs.get("name", self.func.__name__)

        def __rrshift__(self, other):
            # >>
            with tf.variable_scope(self.name, None, self.args):
                out = self.func(other, *self.args, **self.kwargs)
            tf.logging.info(" {} {} {} -> {}".format(
                self.name, "shape", str(other.get_shape()), str(out.get_shape())))
            return out

    return Pipe


@pipe
def identity_layer(tensor, **opts):
    out = tf.identity(tensor)
    return out

<commit_msg>automatic layer renaming sonnet-style by using tf.make_template<commit_after>""" TensorFlow Layers

Convenience functions but Input and Output should be tensors.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from contextlib import contextmanager
from functools import partial, wraps

import tensorflow as tf
from tensorflow.contrib import seq2seq


_phase = tf.Variable(False, name='phase', trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES])
_phase_train = _phase.assign(True)
_phase_infer = _phase.assign(False)


# TODO: move to ops
def _rank(x):
    return len(x.get_shape())


def _apply_dropout_mask(tensor_shape, keep_prob=1.0, normalize=True):
    random_tensor = keep_prob + tf.random_uniform(tensor_shape, dtype=tf.float32)
    binary_mask = tf.floor(random_tensor)
    if normalize:
        binary_mask = tf.reciprocal(keep_prob) * binary_mask
    return binary_mask


def _global_keep_prob(keep_prob):
    keep_prob = tf.convert_to_tensor(keep_prob, dtype=tf.float32)
    keep_prob = tf.cond(_phase, lambda: keep_prob, lambda: keep_prob * 0.0 + 1.0)
    return keep_prob


# TODO: add automatic naming
# TODO: call it layer of pipe?
def pipe(func):

    class Pipe(object):
        def __init__(self, *args, **kwargs):
            self.func = func
            self.args = args
            self.kwargs = kwargs
            self.name = self.kwargs.get("name", self.func.__name__)

            self._template = tf.make_template(self.name, self.func, create_scope_now_=True)
            self._unique_name = self._template.variable_scope.name.split("/")[-1]

        def __rrshift__(self, other):
            # >>
            # with tf.variable_scope(self.name, None, self.args):
                # out = self.func(other, *self.args, **self.kwargs)
            out = self._template(other, *self.args, **self.kwargs)
            tf.logging.info(" {} {} {} -> {}".format(
                self._unique_name, "shape", str(other.get_shape()), str(out.get_shape())))
            return out

        @property
        def template(self):
            return self._template

        @property
        def unique_name(self):
            return self._unique_name

    return Pipe


@pipe
def identity_layer(tensor, **opts):
    out = tf.identity(tensor)
    return out


{'fraction_non_alphanumeric': 0.10762975364745277, 'fraction_numerical': 0.005501076297536474, 'mean_word_length': 3.1653386454183265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1106875', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1264, 'n_words': 312}