# Path: brainforge/learner/backpropagation.py
# class Backpropagation(Learner):
# 
#     def __init__(self, layerstack, cost="mse", optimizer="sgd", name="", **kw):
#         super().__init__(layerstack, cost, name, **kw)
#         self.optimizer = (
#             optimizer if isinstance(optimizer, GradientDescent) else optimizers[optimizer]()
#         )
#         self.optimizer.initialize(nparams=self.layers.num_params)
# 
#     def learn_batch(self, X, Y, w=None, metrics=(), update=True):
#         m = len(X)
#         preds = self.predict(X)
#         delta = self.cost.derivative(preds, Y)
#         if w is not None:
#             delta *= w[:, None]
#         self.backpropagate(delta)
#         if update:
#             self.update(m)
# 
#         train_metrics = {"cost": self.cost(self.output, Y) / m}
#         if metrics:
#             for metric in metrics:
#                 train_metrics[str(metric).lower()] = metric(preds, Y) / m
#         return train_metrics
# 
#     def backpropagate(self, error):
#         for layer in self.layers[-1:0:-1]:
#             error = layer.backpropagate(error)
#         return error
# 
#     def update(self, m):
#         W = self.layers.get_weights(unfold=True)
#         gW = self.get_gradients(unfold=True)
#         oW = self.optimizer.optimize(W, gW, m)
#         self.layers.set_weights(oW, fold=True)
#         self.zero_gradients()
# 
#     def get_weights(self, unfold=True):
#         return self.layers.get_weights(unfold=unfold)
# 
#     def set_weigts(self, ws, fold=True):
#         self.layers.set_weights(ws=ws, fold=fold)
# 
#     def get_gradients(self, unfold=True):
#         grads = [l.gradients for l in self.layers if l.trainable]
#         if unfold:
#             grads = np.concatenate(grads)
#         return grads
# 
#     def set_gradients(self, gradients, fold=True):
#         trl = (l for l in self.layers if l.trainable)
#         if fold:
#             start = 0
#             for layer in trl:
#                 end = start + layer.num_params
#                 layer.set_weights(gradients[start:end])
#                 start = end
#         else:
#             for w, layer in zip(gradients, trl):
#                 layer.set_weights(w)
# 
#     def zero_gradients(self):
#         for layer in self.layers:
#             if not layer.trainable:
#                 continue
#             layer.nabla_w *= 0
#             layer.nabla_b *= 0
# 
#     @property
#     def num_params(self):
#         return self.layers.num_params
# 
#     @property
#     def output_shape(self):
#         return self.layers.output_shape
# 
#     @property
#     def input_shape(self):
#         return self.layers.input_shape
# 
# Path: brainforge/layers/core.py
# class Dense(FFBase):
# 
#     def connect(self, brain):
#         inshape = brain.outshape
#         if len(inshape) != 1:
#             err = "Dense only accepts input shapes with 1 dimension!\n"
#             err += "Maybe you should consider placing <Flatten> before <Dense>?"
#             raise RuntimeError(err)
#         self.weights = white(inshape[0], self.neurons)
#         self.biases = zX(self.neurons)
#         super().connect(brain)
#         if self.compiled:
#             from .. import llatomic
#             print("Compiling DenseLayer...")
#             self.op = llatomic.DenseOp()
#         else:
#             self.op = atomic.DenseOp()
# 
#     def feedforward(self, X):
#         self.inputs = X
#         self.output = self.activation.forward(self.op.forward(
#             X, self.weights, self.biases
#         ))
#         return self.output
# 
#     def backpropagate(self, delta):
#         delta *= self.activation.backward(self.output)
#         nabla_w, nabla_b, dX = self.op.backward(self.inputs, delta, self.weights)
#         self.nabla_w += nabla_w
#         self.nabla_b += nabla_b
#         return dX
# 
#     def __str__(self):
#         return "Dense-{}-{}".format(self.neurons, str(self.activation)[:5])


import numpy as np
from matplotlib import pyplot as plt
from brainforge.learner import Backpropagation
from brainforge.layers import Dense



np.random.seed(1234)

rX = np.linspace(-6., 6., 200)[:, None]
rY = np.sin(rX)

arg = np.arange(len(rX))
np.random.shuffle(arg)
targ, varg = arg[:100], arg[100:]
targ.sort()
varg.sort()

tX, tY = rX[targ], rY[targ]
vX, vY = rX[varg], rY[varg]

tX += np.random.randn(*tX.shape) / np.sqrt(tX.size*0.25)


# Path: brainforge/learner/backpropagation.py
# class Backpropagation(Learner):
# 
#     def __init__(self, layerstack, cost="mse", optimizer="sgd", name="", **kw):
#         super().__init__(layerstack, cost, name, **kw)
#         self.optimizer = (
#             optimizer if isinstance(optimizer, GradientDescent) else optimizers[optimizer]()
#         )
#         self.optimizer.initialize(nparams=self.layers.num_params)
# 
#     def learn_batch(self, X, Y, w=None, metrics=(), update=True):
#         m = len(X)
#         preds = self.predict(X)
#         delta = self.cost.derivative(preds, Y)
#         if w is not None:
#             delta *= w[:, None]
#         self.backpropagate(delta)
#         if update:
#             self.update(m)
# 
#         train_metrics = {"cost": self.cost(self.output, Y) / m}
#         if metrics:
#             for metric in metrics:
#                 train_metrics[str(metric).lower()] = metric(preds, Y) / m
#         return train_metrics
# 
#     def backpropagate(self, error):
#         for layer in self.layers[-1:0:-1]:
#             error = layer.backpropagate(error)
#         return error
# 
#     def update(self, m):
#         W = self.layers.get_weights(unfold=True)
#         gW = self.get_gradients(unfold=True)
#         oW = self.optimizer.optimize(W, gW, m)
#         self.layers.set_weights(oW, fold=True)
#         self.zero_gradients()
# 
#     def get_weights(self, unfold=True):
#         return self.layers.get_weights(unfold=unfold)
# 
#     def set_weigts(self, ws, fold=True):
#         self.layers.set_weights(ws=ws, fold=fold)
# 
#     def get_gradients(self, unfold=True):
#         grads = [l.gradients for l in self.layers if l.trainable]
#         if unfold:
#             grads = np.concatenate(grads)
#         return grads
# 
#     def set_gradients(self, gradients, fold=True):
#         trl = (l for l in self.layers if l.trainable)
#         if fold:
#             start = 0
#             for layer in trl:
#                 end = start + layer.num_params
#                 layer.set_weights(gradients[start:end])
#                 start = end
#         else:
#             for w, layer in zip(gradients, trl):
#                 layer.set_weights(w)
# 
#     def zero_gradients(self):
#         for layer in self.layers:
#             if not layer.trainable:
#                 continue
#             layer.nabla_w *= 0
#             layer.nabla_b *= 0
# 
#     @property
#     def num_params(self):
#         return self.layers.num_params
# 
#     @property
#     def output_shape(self):
#         return self.layers.output_shape
# 
#     @property
#     def input_shape(self):
#         return self.layers.input_shape
# 
# Path: brainforge/layers/core.py
# class Dense(FFBase):
# 
#     def connect(self, brain):
#         inshape = brain.outshape
#         if len(inshape) != 1:
#             err = "Dense only accepts input shapes with 1 dimension!\n"
#             err += "Maybe you should consider placing <Flatten> before <Dense>?"
#             raise RuntimeError(err)
#         self.weights = white(inshape[0], self.neurons)
#         self.biases = zX(self.neurons)
#         super().connect(brain)
#         if self.compiled:
#             from .. import llatomic
#             print("Compiling DenseLayer...")
#             self.op = llatomic.DenseOp()
#         else:
#             self.op = atomic.DenseOp()
# 
#     def feedforward(self, X):
#         self.inputs = X
#         self.output = self.activation.forward(self.op.forward(
#             X, self.weights, self.biases
#         ))
#         return self.output
# 
#     def backpropagate(self, delta):
#         delta *= self.activation.backward(self.output)
#         nabla_w, nabla_b, dX = self.op.backward(self.inputs, delta, self.weights)
#         self.nabla_w += nabla_w
#         self.nabla_b += nabla_b
#         return dX
# 
#     def __str__(self):
#         return "Dense-{}-{}".format(self.neurons, str(self.activation)[:5])


# Path: xperiments/xp_sin.py
import numpy as np
from matplotlib import pyplot as plt
from brainforge.learner import Backpropagation
from brainforge.layers import Dense




np.random.seed(1234)

rX = np.linspace(-6., 6., 200)[:, None]
rY = np.sin(rX)

arg = np.arange(len(rX))
np.random.shuffle(arg)
targ, varg = arg[:100], arg[100:]
targ.sort()
varg.sort()

tX, tY = rX[targ], rY[targ]
vX, vY = rX[varg], rY[varg]

tX += np.random.randn(*tX.shape) / np.sqrt(tX.size*0.25)


net = Backpropagation([Dense(120, activation="tanh"),