<jupyter_start><jupyter_text>Activation Functions<jupyter_code>import numpy as np
def sigmoid(X):
    return 1/(1+np.exp(-X))
def relu(X):
    return np.maximum(0,X)
def softmax(X):
    expo = np.exp(X)
    expo_sum = np.sum(np.exp(X))
    return expo/expo_sum
def tanh(x):
    return np.tanh(x)
def lrelu(X, alpha):
    return np.where(X > 0, X, X * alpha)
def heaviside(X, t):
    return np.where((X-t) != 0, np.maximum(0, np.sign(X-t)), 0.5) 
inp = np.array([
    [1, 0.5, 0.2],
    [-1, -0.5, -0.2],
    [0.1, -0.1, 0]
])
sigmoid(inp)
relu(inp)
softmax(inp)
tanh(inp)
lrelu(inp, 0.1)
heaviside(inp, 0.1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20251177394034536, 'fraction_numerical': 0.0423861852433281, 'mean_word_length': 3.4615384615384617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '496339', 'license': 'MIT', 'n_tokens_mistral': 331, 'n_tokens_neox': 317, 'n_words': 62, 'path': '.ipynb_checkpoints/0-activation_functions-checkpoint.ipynb', 'repo_name': 'Sblbl/ML_implementations', 'size': '637'}