<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

def prelu(features, initializer=None, scope=None):
    """
    Implementation of [Parametric ReLU](https://arxiv.org/abs/1502.01852) borrowed from Keras.
    """
    with tf.variable_scope(scope, 'PReLU', initializer=initializer):
        alpha = tf.get_variable('alpha',
            shape=features.get_shape().as_list()[1:])
        pos = tf.nn.relu(features)
        neg = alpha * (features - tf.abs(features)) * 0.5
        return pos + neg
<commit_msg>Clip alpha of prelu to avoid NaN<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

def prelu(features, initializer=None, scope=None):
    """
    Implementation of [Parametric ReLU](https://arxiv.org/abs/1502.01852) borrowed from Keras.
    """
    with tf.variable_scope(scope, 'PReLU', initializer=initializer):
        alpha = tf.get_variable('alpha',
            shape=features.get_shape().as_list()[1:])
        pos = tf.nn.relu(features)
        neg = tf.clip_by_value(alpha, 0, 1) * (features - tf.abs(features)) * 0.5 # TODO: Clipping may not be necessary
        return pos + neg

{'fraction_non_alphanumeric': 0.1136890951276102, 'fraction_numerical': 0.020108275328692964, 'mean_word_length': 3.6379928315412187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1338294', 'n_tokens_mistral': 419, 'n_tokens_neox': 393, 'n_words': 102}