references https en wikipedia orgwikiactivationfunction avoid numerical overflow by removing max smooth relu avoid numerical overflow see https docs scipy orgdocnumpyreferencegeneratednumpy logaddexp html return np logaddexp0 0 z def softsignz return z 1 np absz def tanhz return np tanhz def reluz return np maximum0 z def leakyreluz a0 01 return np maximumz a z def getactivationname references https en wikipedia org wiki activation_function avoid numerical overflow by removing max smooth relu avoid numerical overflow see https docs scipy org doc numpy reference generated numpy logaddexp html return activation function by name
import autograd.numpy as np




def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-z))


def softmax(z):
    
    e = np.exp(z - np.amax(z, axis=1, keepdims=True))
    return e / np.sum(e, axis=1, keepdims=True)


def linear(z):
    return z


def softplus(z):
    
    
    
    return np.logaddexp(0.0, z)


def softsign(z):
    return z / (1 + np.abs(z))


def tanh(z):
    return np.tanh(z)


def relu(z):
    return np.maximum(0, z)


def leakyrelu(z, a=0.01):
    return np.maximum(z * a, z)


def get_activation(name):
    
    try:
        return globals()[name]
    except Exception:
        raise ValueError("Invalid activation function.")