coding utf8 allocates initial weights pass def forwardpassself x raise notimplementederror def backwardpassself delta raise notimplementederror def shapeself xshape a fully connected layer parameters outputdim int update gradient values derivative of activation function randomly set a fraction of p inputs to 0 at each training update def initself p0 1 self p p self mask none def forwardpassself x assert self p 0 if self istraining self mask np random uniformsizex shape self p y x self mask else y x 1 0 self p return y def backwardpassself delta return delta self mask def shapeself xshape return xshape class timestepslicerlayer apply regular dense layer to every timestep def initself outputdim self outputdim outputdim self ntimesteps none self dense none self inputdim none def setupself xshape self dense denseself outputdim self dense setupxshape0 xshape2 self inputdim xshape2 def forwardpassself x ntimesteps x shape1 x x reshape1 x shape1 y self dense forwardpassx y y reshape1 ntimesteps self outputdim return y def backwardpassself delta ntimesteps delta shape1 x delta reshape1 delta shape1 y self dense backwardpassx y y reshape1 ntimesteps self inputdim return y property def parametersself return self dense params def shapeself xshape return xshape0 xshape1 self outputdim coding utf 8 allocates initial weights returns shape of the current layer a fully connected layer parameters output_dim int update gradient values derivative of activation function randomly set a fraction of p inputs to 0 at each training update take a specific time step from 3d tensor apply regular dense layer to every timestep
import autograd.numpy as np
from autograd import elementwise_grad

from mla.neuralnet.activations import get_activation
from mla.neuralnet.parameters import Parameters

np.random.seed(9999)


class Layer(object):
    def setup(self, X_shape):
        
        pass

    def forward_pass(self, x):
        raise NotImplementedError()

    def backward_pass(self, delta):
        raise NotImplementedError()

    def shape(self, x_shape):
        
        raise NotImplementedError()


class ParamMixin(object):
    @property
    def parameters(self):
        return self._params


class PhaseMixin(object):
    _train = False

    @property
    def is_training(self):
        return self._train

    @is_training.setter
    def is_training(self, is_train=True):
        self._train = is_train

    @property
    def is_testing(self):
        return not self._train

    @is_testing.setter
    def is_testing(self, is_test=True):
        self._train = not is_test


class Dense(Layer, ParamMixin):
    def __init__(self, output_dim, parameters=None):
        
        self._params = parameters
        self.output_dim = output_dim
        self.last_input = None

        if parameters is None:
            self._params = Parameters()

    def setup(self, x_shape):
        self._params.setup_weights((x_shape[1], self.output_dim))

    def forward_pass(self, X):
        self.last_input = X
        return self.weight(X)

    def weight(self, X):
        W = np.dot(X, self._params["W"])
        return W + self._params["b"]

    def backward_pass(self, delta):
        dW = np.dot(self.last_input.T, delta)
        db = np.sum(delta, axis=0)

        
        self._params.update_grad("W", dW)
        self._params.update_grad("b", db)
        return np.dot(delta, self._params["W"].T)

    def shape(self, x_shape):
        return x_shape[0], self.output_dim


class Activation(Layer):
    def __init__(self, name):
        self.last_input = None
        self.activation = get_activation(name)
        
        self.activation_d = elementwise_grad(self.activation)

    def forward_pass(self, X):
        self.last_input = X
        return self.activation(X)

    def backward_pass(self, delta):
        return self.activation_d(self.last_input) * delta

    def shape(self, x_shape):
        return x_shape


class Dropout(Layer, PhaseMixin):
    

    def __init__(self, p=0.1):
        self.p = p
        self._mask = None

    def forward_pass(self, X):
        assert self.p > 0
        if self.is_training:
            self._mask = np.random.uniform(size=X.shape) > self.p
            y = X * self._mask
        else:
            y = X * (1.0 - self.p)

        return y

    def backward_pass(self, delta):
        return delta * self._mask

    def shape(self, x_shape):
        return x_shape


class TimeStepSlicer(Layer):
    

    def __init__(self, step=-1):
        self.step = step

    def forward_pass(self, x):
        return x[:, self.step, :]

    def backward_pass(self, delta):
        return np.repeat(delta[:, np.newaxis, :], 2, 1)

    def shape(self, x_shape):
        return x_shape[0], x_shape[2]


class TimeDistributedDense(Layer):
    

    def __init__(self, output_dim):
        self.output_dim = output_dim
        self.n_timesteps = None
        self.dense = None
        self.input_dim = None

    def setup(self, X_shape):
        self.dense = Dense(self.output_dim)
        self.dense.setup((X_shape[0], X_shape[2]))
        self.input_dim = X_shape[2]

    def forward_pass(self, X):
        n_timesteps = X.shape[1]
        X = X.reshape(-1, X.shape[-1])
        y = self.dense.forward_pass(X)
        y = y.reshape((-1, n_timesteps, self.output_dim))
        return y

    def backward_pass(self, delta):
        n_timesteps = delta.shape[1]
        X = delta.reshape(-1, delta.shape[-1])
        y = self.dense.backward_pass(X)
        y = y.reshape((-1, n_timesteps, self.input_dim))
        return y

    @property
    def parameters(self):
        return self.dense._params

    def shape(self, x_shape):
        return x_shape[0], x_shape[1], self.output_dim