applies the rectified linear unit activation function with default values this returns the standard relu activation maxx 0 the elementwise maximum of 0 and the input tensor modifying default parameters allows you to use nonzero thresholds change the max value of the activation and to use a nonzero multiple of the input for values below the threshold examples x 10 5 0 0 5 10 keras activations relux 0 0 0 5 10 keras activations relux negativeslope0 5 5 2 5 0 5 10 keras activations relux maxvalue5 0 0 0 5 5 keras activations relux threshold5 0 0 0 0 10 args x input tensor negativeslope a float that controls the slope for values lower than the threshold maxvalue a float that sets the saturation threshold the largest value the function will return threshold a float giving the threshold value of the activation function below which values will be damped or set to zero returns a tensor with the same shape and dtype as input x computes x for x threshold else 0 if no threshold then can use nn relu6 native op for performance leaky relu activation function args x input tensor negativeslope a float that controls the slope for values lower than the threshold relu6 activation function it s the relu function but truncated to a maximum value of 6 args x input tensor softmax converts a vector of values to a probability distribution the elements of the output vector are in range 0 1 and sum to 1 each input vector is handled independently the axis argument sets which axis of the input the function is applied along softmax is often used as the activation for the last layer of a classification network because the result could be interpreted as a probability distribution the softmax of each vector x is computed as expx sumexpx the input values in are the logodds of the resulting probability args x input tensor axis integer axis along which the softmax is applied cache the logits to use for crossentropy loss we re dealing with a ctype exponential linear unit the exponential linear unit elu with alpha 0 is define as x if x 0 alpha expx 1 if x 0 elus have negative values which pushes the mean of the activations closer to zero mean activations that are closer to zero enable faster learning as they bring the gradient closer to the natural gradient elus saturate to a negative value when the argument gets smaller saturation means a small derivative which decreases the variation and the information that is propagated to the next layer args x input tensor reference clevert et al 2016https arxiv orgabs1511 07289 scaled exponential linear unit selu the scaled exponential linear unit selu activation function is defined as scale x if x 0 scale alpha expx 1 if x 0 where alpha and scale are predefined constants alpha1 67326324 and scale1 05070098 basically the selu activation function multiplies scale 1 with the output of the keras activations elu function to ensure a slope larger than one for positive inputs the values of alpha and scale are chosen so that the mean and variance of the inputs are preserved between two consecutive layers as long as the weights are initialized correctly see keras initializers lecunnormal initializer and the number of input units is large enough see reference paper for more information args x input tensor notes to be used together with the keras initializers lecunnormal initializer to be used together with the dropout variant keras layers alphadropout rather than regular dropout reference klambauer et al 2017https arxiv orgabs1706 02515 softplus activation function it is defined as softplusx logexpx 1 args x input tensor softsign activation function softsign is defined as softsignx x absx 1 args x input tensor swish or silu activation function it is defined as swishx x sigmoidx the swish or silu activation function is a smooth nonmonotonic function that is unbounded above and bounded below args x input tensor reference ramachandran et al 2017https arxiv orgabs1710 05941 gaussian error linear unit gelu activation function the gaussian error linear unit gelu is defined as gelux x px x where px n0 1 i e gelux 0 5 x 1 erfx sqrt2 gelu weights inputs by their value rather than gating inputs by their sign as in relu args x input tensor approximate a bool whether to enable approximation reference hendrycks et al 2016https arxiv orgabs1606 08415 hyperbolic tangent activation function it is defined as tanhx sinhx coshx i e tanhx expx expx expx expx args x input tensor sigmoid activation function it is defined as sigmoidx 1 1 expx for small values 5 sigmoid returns a value close to zero and for large values 5 the result of the function gets close to 1 sigmoid is equivalent to a 2element softmax where the second element is assumed to be zero the sigmoid function always returns a value between 0 and 1 args x input tensor cache the logits to use for crossentropy loss we re dealing with a ctype exponential activation function args x input tensor hard sigmoid activation function the hard sigmoid activation is defined as 0 if if x 2 5 1 if x 2 5 0 2 x 0 5 if 2 5 x 2 5 it s a faster piecewise linear approximation of the sigmoid activation args x input tensor reference wikipedia hard sigmoidhttps en wikipedia orgwikihardsigmoid hard swish activation function the hard swish activation is defined as 0 if if x 3 x if x 3 x x 3 6 if 3 x 3 it s a faster piecewise linear approximation of the swish activation args x input tensor reference a howard 2019https arxiv orgabs1905 02244 linear activation function passthrough a linear activation is an identity function it returns the input unmodified args x input tensor mish activation function it is defined as mishx x tanhsoftplusx where softplus is defined as softplusx logexpx 1 args x input tensor reference misra 2019https arxiv orgabs1908 08681 logsoftmax activation function each input vector is handled independently the axis argument sets which axis of the input the function is applied along args x input tensor axis integer axis along which the softmax is applied applies the rectified linear unit activation function with default values this returns the standard relu activation max x 0 the element wise maximum of 0 and the input tensor modifying default parameters allows you to use non zero thresholds change the max value of the activation and to use a non zero multiple of the input for values below the threshold examples x 10 5 0 0 5 10 keras activations relu x 0 0 0 5 10 keras activations relu x negative_slope 0 5 5 2 5 0 5 10 keras activations relu x max_value 5 0 0 0 5 5 keras activations relu x threshold 5 0 0 0 0 10 args x input tensor negative_slope a float that controls the slope for values lower than the threshold max_value a float that sets the saturation threshold the largest value the function will return threshold a float giving the threshold value of the activation function below which values will be damped or set to zero returns a tensor with the same shape and dtype as input x computes x for x threshold else 0 if no threshold then can use nn relu6 native op for performance leaky relu activation function args x input tensor negative_slope a float that controls the slope for values lower than the threshold relu6 activation function it s the relu function but truncated to a maximum value of 6 args x input tensor softmax converts a vector of values to a probability distribution the elements of the output vector are in range 0 1 and sum to 1 each input vector is handled independently the axis argument sets which axis of the input the function is applied along softmax is often used as the activation for the last layer of a classification network because the result could be interpreted as a probability distribution the softmax of each vector x is computed as exp x sum exp x the input values in are the log odds of the resulting probability args x input tensor axis integer axis along which the softmax is applied cache the logits to use for crossentropy loss we re dealing with a c type exponential linear unit the exponential linear unit elu with alpha 0 is define as x if x 0 alpha exp x 1 if x 0 elus have negative values which pushes the mean of the activations closer to zero mean activations that are closer to zero enable faster learning as they bring the gradient closer to the natural gradient elus saturate to a negative value when the argument gets smaller saturation means a small derivative which decreases the variation and the information that is propagated to the next layer args x input tensor reference clevert et al 2016 https arxiv org abs 1511 07289 scaled exponential linear unit selu the scaled exponential linear unit selu activation function is defined as scale x if x 0 scale alpha exp x 1 if x 0 where alpha and scale are pre defined constants alpha 1 67326324 and scale 1 05070098 basically the selu activation function multiplies scale 1 with the output of the keras activations elu function to ensure a slope larger than one for positive inputs the values of alpha and scale are chosen so that the mean and variance of the inputs are preserved between two consecutive layers as long as the weights are initialized correctly see keras initializers lecunnormal initializer and the number of input units is large enough see reference paper for more information args x input tensor notes to be used together with the keras initializers lecunnormal initializer to be used together with the dropout variant keras layers alphadropout rather than regular dropout reference klambauer et al 2017 https arxiv org abs 1706 02515 softplus activation function it is defined as softplus x log exp x 1 args x input tensor softsign activation function softsign is defined as softsign x x abs x 1 args x input tensor swish or silu activation function it is defined as swish x x sigmoid x the swish or silu activation function is a smooth non monotonic function that is unbounded above and bounded below args x input tensor reference ramachandran et al 2017 https arxiv org abs 1710 05941 gaussian error linear unit gelu activation function the gaussian error linear unit gelu is defined as gelu x x p x x where p x n 0 1 i e gelu x 0 5 x 1 erf x sqrt 2 gelu weights inputs by their value rather than gating inputs by their sign as in relu args x input tensor approximate a bool whether to enable approximation reference hendrycks et al 2016 https arxiv org abs 1606 08415 hyperbolic tangent activation function it is defined as tanh x sinh x cosh x i e tanh x exp x exp x exp x exp x args x input tensor sigmoid activation function it is defined as sigmoid x 1 1 exp x for small values 5 sigmoid returns a value close to zero and for large values 5 the result of the function gets close to 1 sigmoid is equivalent to a 2 element softmax where the second element is assumed to be zero the sigmoid function always returns a value between 0 and 1 args x input tensor cache the logits to use for crossentropy loss we re dealing with a c type exponential activation function args x input tensor hard sigmoid activation function the hard sigmoid activation is defined as 0 if if x 2 5 1 if x 2 5 0 2 x 0 5 if 2 5 x 2 5 it s a faster piecewise linear approximation of the sigmoid activation args x input tensor reference wikipedia hard sigmoid https en wikipedia org wiki hard_sigmoid hard swish activation function the hard swish activation is defined as 0 if if x 3 x if x 3 x x 3 6 if 3 x 3 it s a faster piecewise linear approximation of the swish activation args x input tensor reference a howard 2019 https arxiv org abs 1905 02244 linear activation function pass through a linear activation is an identity function it returns the input unmodified args x input tensor mish activation function it is defined as mish x x tanh softplus x where softplus is defined as softplus x log exp x 1 args x input tensor reference misra 2019 https arxiv org abs 1908 08681 log softmax activation function each input vector is handled independently the axis argument sets which axis of the input the function is applied along args x input tensor axis integer axis along which the softmax is applied
from keras import backend
from keras import ops
from keras.api_export import keras_export


@keras_export("keras.activations.relu")
def relu(x, negative_slope=0.0, max_value=None, threshold=0.0):
    
    if backend.any_symbolic_tensors((x,)):
        return ReLU(
            negative_slope=negative_slope,
            max_value=max_value,
            threshold=threshold,
        )(x)
    return ReLU.static_call(
        x,
        negative_slope=negative_slope,
        max_value=max_value,
        threshold=threshold,
    )


class ReLU(ops.Operation):
    def __init__(
        self, negative_slope=0.0, max_value=None, threshold=0.0, name=None
    ):
        super().__init__(name=name)
        self.negative_slope = negative_slope
        self.max_value = max_value
        self.threshold = threshold

    def call(self, x):
        return self.static_call(
            x,
            negative_slope=self.negative_slope,
            max_value=self.max_value,
            threshold=self.threshold,
        )

    def compute_output_spec(self, x):
        return backend.KerasTensor(x.shape, x.dtype)

    @staticmethod
    def static_call(x, negative_slope=0.0, max_value=None, threshold=0.0):
        x = backend.convert_to_tensor(x)
        if negative_slope != 0.0:
            if max_value is None and threshold == 0:
                return backend.nn.leaky_relu(x, negative_slope=negative_slope)

            if threshold != 0:
                negative_part = backend.nn.relu(-x + threshold)
            else:
                negative_part = backend.nn.relu(-x)

        clip_max = max_value is not None
        if threshold != 0:
            
            threshold = ops.cast(threshold, dtype=x.dtype)
            x = x * backend.cast(
                backend.numpy.greater(x, threshold), dtype=x.dtype
            )
        elif max_value == 6:
            
            x = backend.nn.relu6(x)
            clip_max = False
        else:
            x = backend.nn.relu(x)

        if clip_max:
            min_value = ops.cast(0.0, dtype=x.dtype)
            max_value = ops.cast(max_value, dtype=x.dtype)
            x = backend.numpy.clip(x, min_value, max_value)

        if negative_slope != 0.0:
            x -= negative_slope * negative_part
        return x


@keras_export("keras.activations.leaky_relu")
def leaky_relu(x, negative_slope=0.2):
    
    return ops.leaky_relu(x, negative_slope=negative_slope)


@keras_export("keras.activations.relu6")
def relu6(x):
    
    return ops.relu6(x)


@keras_export("keras.activations.softmax")
def softmax(x, axis=-1):
    
    output = ops.softmax(x, axis=axis)
    
    try:
        output._keras_logits = x
    except AttributeError:
        
        pass
    return output


@keras_export("keras.activations.elu")
def elu(x, alpha=1.0):
    
    return ops.elu(x, alpha=alpha)


@keras_export("keras.activations.selu")
def selu(x):
    
    return ops.selu(x)


@keras_export("keras.activations.softplus")
def softplus(x):
    
    return ops.softplus(x)


@keras_export("keras.activations.softsign")
def softsign(x):
    
    return ops.softsign(x)


@keras_export(["keras.activations.silu", "keras.activations.swish"])
def silu(x):
    
    return ops.silu(x)


@keras_export("keras.activations.gelu")
def gelu(x, approximate=False):
    
    return ops.gelu(x, approximate=approximate)


@keras_export("keras.activations.tanh")
def tanh(x):
    
    return ops.tanh(x)


@keras_export("keras.activations.sigmoid")
def sigmoid(x):
    
    output = ops.sigmoid(x)
    
    try:
        output._keras_logits = x
    except AttributeError:
        
        pass
    return output


@keras_export("keras.activations.exponential")
def exponential(x):
    
    return ops.exp(x)


@keras_export("keras.activations.hard_sigmoid")
def hard_sigmoid(x):
    
    return ops.hard_sigmoid(x)


@keras_export("keras.activations.hard_swish")
def hard_swish(x):
    
    x = backend.convert_to_tensor(x)
    return ops.hard_swish(x)


@keras_export("keras.activations.linear")
def linear(x):
    
    return x


class Mish(ops.Operation):
    def call(self, x):
        return self.static_call(x)

    def compute_output_spec(self, x):
        return backend.KerasTensor(x.shape, x.dtype)

    @staticmethod
    def static_call(x):
        return x * backend.nn.tanh(backend.nn.softplus(x))


@keras_export("keras.activations.mish")
def mish(x):
    
    x = backend.convert_to_tensor(x)
    return Mish.static_call(x)


@keras_export("keras.activations.log_softmax")
def log_softmax(x, axis=-1):
    
    return ops.log_softmax(x, axis=axis)