# Path: mlfromscratch/deep_learning/activation_functions.py
# class Sigmoid():
#     def __call__(self, x):
#         return 1 / (1 + np.exp(-x))
# 
#     def gradient(self, x):
#         return self.__call__(x) * (1 - self.__call__(x))
# 
# class ReLU():
#     def __call__(self, x):
#         return np.where(x >= 0, x, 0)
# 
#     def gradient(self, x):
#         return np.where(x >= 0, 1, 0)
# 
# class SoftPlus():
#     def __call__(self, x):
#         return np.log(1 + np.exp(x))
# 
#     def gradient(self, x):
#         return 1 / (1 + np.exp(-x))
# 
# class LeakyReLU():
#     def __init__(self, alpha=0.2):
#         self.alpha = alpha
# 
#     def __call__(self, x):
#         return np.where(x >= 0, x, self.alpha * x)
# 
#     def gradient(self, x):
#         return np.where(x >= 0, 1, self.alpha)
# 
# Path: mlfromscratch/deep_learning/activation_functions.py
# class TanH():
#     def __call__(self, x):
#         return 2 / (1 + np.exp(-2*x)) - 1
# 
#     def gradient(self, x):
#         return 1 - np.power(self.__call__(x), 2)
# 
# class ELU():
#     def __init__(self, alpha=0.1):
#         self.alpha = alpha 
# 
#     def __call__(self, x):
#         return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
# 
#     def gradient(self, x):
#         return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)
# 
# class SELU():
#     # Reference : https://arxiv.org/abs/1706.02515,
#     # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
#     def __init__(self):
#         self.alpha = 1.6732632423543772848170429916717
#         self.scale = 1.0507009873554804934193349852946 
# 
#     def __call__(self, x):
#         return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))
# 
#     def gradient(self, x):
#         return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))
# 
# class Softmax():
#     def __call__(self, x):
#         e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
#         return e_x / np.sum(e_x, axis=-1, keepdims=True)
# 
#     def gradient(self, x):
#         p = self.__call__(x)
#         return p * (1 - p)


import math
import numpy as np
import copy
from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, LeakyReLU
from mlfromscratch.deep_learning.activation_functions import TanH, ELU, SELU, Softmax

    Parameters:
    -----------
    p: float
        The probability that unit x is set to zero.
    """
    def __init__(self, p=0.2):
        self.p = p
        self._mask = None
        self.input_shape = None
        self.n_units = None
        self.pass_through = True
        self.trainable = True

    def forward_pass(self, X, training=True):
        c = (1 - self.p)
        if training:
            self._mask = np.random.uniform(size=X.shape) > self.p
            c = self._mask
        return X * c

    def backward_pass(self, accum_grad):
        return accum_grad * self._mask

    def output_shape(self):
        return self.input_shape

activation_functions = {
    'relu': ReLU,
    'sigmoid': Sigmoid,

# Path: mlfromscratch/deep_learning/activation_functions.py
# class Sigmoid():
#     def __call__(self, x):
#         return 1 / (1 + np.exp(-x))
# 
#     def gradient(self, x):
#         return self.__call__(x) * (1 - self.__call__(x))
# 
# class ReLU():
#     def __call__(self, x):
#         return np.where(x >= 0, x, 0)
# 
#     def gradient(self, x):
#         return np.where(x >= 0, 1, 0)
# 
# class SoftPlus():
#     def __call__(self, x):
#         return np.log(1 + np.exp(x))
# 
#     def gradient(self, x):
#         return 1 / (1 + np.exp(-x))
# 
# class LeakyReLU():
#     def __init__(self, alpha=0.2):
#         self.alpha = alpha
# 
#     def __call__(self, x):
#         return np.where(x >= 0, x, self.alpha * x)
# 
#     def gradient(self, x):
#         return np.where(x >= 0, 1, self.alpha)
# 
# Path: mlfromscratch/deep_learning/activation_functions.py
# class TanH():
#     def __call__(self, x):
#         return 2 / (1 + np.exp(-2*x)) - 1
# 
#     def gradient(self, x):
#         return 1 - np.power(self.__call__(x), 2)
# 
# class ELU():
#     def __init__(self, alpha=0.1):
#         self.alpha = alpha 
# 
#     def __call__(self, x):
#         return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
# 
#     def gradient(self, x):
#         return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)
# 
# class SELU():
#     # Reference : https://arxiv.org/abs/1706.02515,
#     # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
#     def __init__(self):
#         self.alpha = 1.6732632423543772848170429916717
#         self.scale = 1.0507009873554804934193349852946 
# 
#     def __call__(self, x):
#         return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))
# 
#     def gradient(self, x):
#         return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))
# 
# class Softmax():
#     def __call__(self, x):
#         e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
#         return e_x / np.sum(e_x, axis=-1, keepdims=True)
# 
#     def gradient(self, x):
#         p = self.__call__(x)
#         return p * (1 - p)


# Path: mlfromscratch/deep_learning/layers.py
import math
import numpy as np
import copy
from mlfromscratch.deep_learning.activation_functions import Sigmoid, ReLU, SoftPlus, LeakyReLU
from mlfromscratch.deep_learning.activation_functions import TanH, ELU, SELU, Softmax


    Parameters:
    -----------
    p: float
        The probability that unit x is set to zero.
    """
    def __init__(self, p=0.2):
        self.p = p
        self._mask = None
        self.input_shape = None
        self.n_units = None
        self.pass_through = True
        self.trainable = True

    def forward_pass(self, X, training=True):
        c = (1 - self.p)
        if training:
            self._mask = np.random.uniform(size=X.shape) > self.p
            c = self._mask
        return X * c

    def backward_pass(self, accum_grad):
        return accum_grad * self._mask

    def output_shape(self):
        return self.input_shape

activation_functions = {
    'relu': ReLU,
    'sigmoid': Sigmoid,

    'selu': SELU,