<jupyter_start><jupyter_text>"1.5 Introdução a Redes Neurais: minha primeira Rede Neural <3"Vamos codar nossa primeira rede neural que de fato aprende com exemplos. Para isso vamos começar resolvendo o problema de regressão e classificação com bases de dados simples e fictícias para então resolver uma base de dados real.  Dependências<jupyter_code>#hide
# Gráficos
import matplotlib.pyplot as plt

# Matemática + manipulação de vetores
import math
import numpy as np

# "Fixar" números aleatórios a serem gerados
np.random.seed(0)

# Trabalhar com os dados
import pandas as pd
from sklearn.datasets import fetch_mldata
from sklearn.model_selection import train_test_split

# Utilidades
import utils

# Recarregar automaticamente dependências caso elas mudem
%load_ext autoreload
%autoreload 2

np.seterr(all='raise')<jupyter_output><empty_output><jupyter_text>Gerando dados sintéticos$Y = 7 * X + 15$<jupyter_code># número de dados sintéticos gerados
SYNT_TRAIN_SIZE = 200

# controla o quão espalhados são os dados
STD_DEV = 0.7

def random_error(size, mu=0, std_dev=0.5):
    return np.random.normal(mu, std_dev, size)

def add_batch_dim(tensor):
    if len(tensor.shape) == 1:
        return np.expand_dims(tensor, axis=1)
    else:
        return tensor

def remove_batch_dim(tensor):
    return np.squeeze(tensor, axis=1)
    
def generate_x(size, use_batch_dim=True):
    x = np.random.rand(size)
    if use_batch_dim:
        x = add_batch_dim(x)
    return x

def plot_line(x, y, style='-b'):
    x, y = remove_batch_dim(x), remove_batch_dim(y)
    return plt.plot([min(x), max(x)], [min(y), max(y)], style)

def generate_f(x, a=7, b=15, error_std_dev=0.5, use_batch_dim=True):
    y = a * x + b + random_error(x.shape, std_dev=error_std_dev)
    if use_batch_dim:
        y = add_batch_dim(y)
    return y

# gera valores aleatórios para x
synt_x = generate_x(SYNT_TRAIN_SIZE)

# gera a funcão: Y = 7 * X + 15
synt_y = generate_f(synt_x, error_std_dev=STD_DEV)
plt.plot(synt_x, synt_y, 'ro', alpha=0.4)
plot_line(synt_x, synt_x * 7 + 15)
plt.show()<jupyter_output><empty_output><jupyter_text>Implementando Rede Neural<jupyter_code>class NeuralNetwork(object):
    def __init__(self, layers=[1], input_size=1, activations=[None]):
        assert len(layers) == len(activations)
        self.input_size = input_size
        self.layers = layers
        self.activations, self._act_devs = self.get_act(activations)
        
        self.weights, self.biases = self.define_params()
        self._current_batch = []
        
    def get_act(self, act_names):
        def _no_act(x):
            return x
        def _dev_no_act(x):
            return np.ones(x.shape)

        def _sigmoid(x):
            return 1 / (1 + np.exp(-x))
        
        def _dev_sigmoid(x):
            return x * (1 - x)
        
        def _relu(x):
            return np.maximum(1e-15, x)
        
        def _dev_relu(x):
            return (x > 0) * 1.0
        
        activations = []
        act_devs = []
        for act_name in act_names:
            if act_name is None:
                act, dev_act = _no_act, _dev_no_act
            elif act_name == 'sigmoid':
                act, dev_act = _sigmoid, _dev_sigmoid
            elif act_name == 'relu':
                act, dev_act = _relu, _dev_relu
            else:
                raise ValueError('Activation function is not valid: %s' % act_name)
            
            activations.append(act)
            act_devs.append(dev_act)
        return activations, act_devs
    

    def define_params(self):
        '''He-et-all initialization'''
        weights = []
        biases = []
        for i, (in_dim, out_dim) in enumerate(zip([self.input_size] + self.layers, self.layers)):
            weights.append(np.random.randn(in_dim, out_dim) * np.sqrt(2/in_dim))
            biases.append(np.random.randn(out_dim) * np.sqrt(2/in_dim))           
            
        return weights, biases


    def update_params(self, gradients, learning_rate=0.1):
        assert len(gradients) == len(self.weights), (len(gradients), len(self.weights))
        assert len(gradients) == len(self.biases), (len(gradients), len(self.biases))
        
        for i, grad in enumerate(gradients[::-1]):
            assert grad['weights'].shape == self.weights[i].shape
            self.weights[i] -= learning_rate * grad['weights']
            self.biases[i] -= learning_rate * grad['biases']

    
    def run_batch(self, batch):
        self._current_batch = [batch]
        for i, (w, b) in enumerate(zip(self.weights, self.biases)):
            output = np.dot(self._current_batch[-1], w) + b
            output = self.activations[i](output)
            self._current_batch.append(output)
        
        self._current_batch = self._current_batch[::-1]
        return output<jupyter_output><empty_output><jupyter_text>Implementando SGD<jupyter_code>class Trainer(object):
    def __init__(self, model, learning_rate = 0.01, loss_name='l2',
                 print_mod=1000, verbose=True):
        
        def _accuracy(pred_y, real_y):
            p = np.argmax(self.softmax(pred_y), axis=1)
            return np.sum(p == real_y) / len(pred_y)
            
        
        self.model = model
        self.loss_name = loss_name
        self.learning_rate = learning_rate
        self.loss, self.loss_dev = self._define_loss()
        
        self.train_step = 0
        self.eval_steps = []
        
        self.verbose = verbose
        self.print_mod = print_mod
        
        self.train_losses = []
        self.eval_losses = []
        
        self._metrics = {
            'accuracy': _accuracy
        }
    
    def softmax(self, x):
        exps = np.exp(x)
        return (exps / np.sum(exps, axis=1, keepdims=True))
        
    def _define_loss(self):
        def _l2(pred_y, real_y):
            n = len(pred_y)
            return (1.0/2) * (1.0/n) * np.sum(np.power(pred_y - real_y, 2))
        
        def _l2_dev(pred_y, real_y):
            n = len(pred_y)
            return (pred_y - real_y) * (1.0/n)
        
        def _cross_entropy(pred_y, real_y):
            m = real_y.shape[0]
            p = self.softmax(pred_y)
            # We use multidimensional array indexing to extract 
            # softmax probability of the correct label for each sample.
            # Refer to https://docs.scipy.org/doc/numpy/user/basics.indexing.html#indexing-multi-dimensional-arrays for understanding multidimensional array indexing.
            log_likelihood = -np.log(p[range(m), real_y.astype(int)])
            loss = np.sum(log_likelihood) / m
            return loss
    
        
        def _cross_entropy_dev(pred_y, real_y):
            m = real_y.shape[0]
            grad = self.softmax(pred_y)
            grad[range(m), real_y.astype(int)] -= 1
            grad = grad / m
            return grad

        if self.loss_name == 'l2':
            return _l2, _l2_dev
        elif self.loss_name == 'cross-entropy':
            return _cross_entropy, _cross_entropy_dev
        else:
            raise ValueError('Invalid loss name: %s' % self.loss_name)


    def train(self, batch_x, batch_y):
        self.train_step += 1
        
        # run feed forward network
        pred_y = self.model.run_batch(batch_x)
        # save loss
        self.train_losses.append(self.loss(pred_y, batch_y))
        # get gradients
        grads = self.generate_gradients(pred_y, batch_y, batch_x)
        # update parameters
        self.model.update_params(grads, self.learning_rate)

        if self.verbose and (self.train_step - 1) % self.print_mod == 0:
            print('Loss: %.4f for step %d' % (self.train_losses[-1], self.train_step))


    def eval(self, batch_x, batch_y, metrics=[]):
        # run feed forward network
        pred_y = self.model.run_batch(batch_x)
        # loss
        loss = self.loss(pred_y, batch_y)
        self.eval_losses.append(loss)
        # metrics
        res_metrics = []
        for m in metrics:
            if m in self._metrics:
                res_metrics.append(self._metrics[m](pred_y, batch_y))
            else:
                raise ValueError('Invalid metric: %s' % m)
        
        self.eval_steps.append(self.train_step)
            
        return loss, res_metrics

    
    def plot_losses(self):
        if len(self.eval_losses) > 0:
            plt.title('Train Loss: %.4f | Test Loss: %.4f for step %d' % (self.train_losses[-1], self.eval_losses[-1], self.train_step))
        else:
            plt.title('Train Loss: %.4f for step %d' % (self.train_losses[-1], self.train_step))    
        plt.plot([i for i in range(self.train_step)], self.train_losses)
        plt.plot([i for i in self.eval_steps], self.eval_losses)
        
        
    def generate_gradients(self, pred_y, real_y, data_x):
        grad = []
        input_size = pred_y.shape[0]
        j = len(self.model.activations) - 1
        k = len(self.model.weights) - 1
        dly = self.loss_dev(pred_y, real_y) * self.model._act_devs[j](self.model._current_batch[0])
        dlx = np.dot(dly, self.model.weights[k].T)

        for i, (w, b) in enumerate(zip(self.model.weights[::-1], self.model.biases[::-1])):
            dlw = np.dot(self.model._current_batch[i+1].T, dly)
            dlb = np.sum(dly)
            # print('weight:', w.shape, 'bias:', b.shape)
            # print('dlw:', dlw.shape, 'dlb:', dlb.shape)
            # print('dly:', dly.shape, 'dlx:', dlx.shape)
            grad.append({
                'weights': dlw,
                'biases': dlb
            })
            
            j -= 1
            k -= 1
            if i != len(self.model.weights)-1:
                dly = dlx * self.model._act_devs[j](self.model._current_batch[i+1])
                dlx = np.dot(dly, self.model.weights[k].T)
        return grad<jupyter_output><empty_output><jupyter_text>Gradients L2 loss with 1 layer, no activation**Loss**$$L = 1/2 * 1/n * \sum{(y_i - ŷ_i)^{2}}$$$$L = 1/2 * 1/n * \sum{(y_i - w_i * x_i + b_i)^{2}}$$**Gradients**$$\frac{\partial L}{\partial w_i} = 1/2 * 1/n * 2 * \sum{(y_i - ŷ_i)} * \frac{\partial {ŷ_i}}{\partial w_i} $$$$\frac{\partial L}{\partial w_i} = 1/n * \sum{(y_i - ŷ_i)} * x_i$$---$$\frac{\partial L}{\partial b_i} = 1/2 * 1/n * 2 * \sum{(y_i - ŷ_i)} * \frac{\partial {ŷ_i}}{\partial b_i} $$$$\frac{\partial L}{\partial b_i} = 1/n * \sum{(y_i - ŷ_i)} * 1$$ L2 loss with 2 layers, relu activation in the hidden layer**Loss**$$L = 1/2 * 1/n * \sum{(y_i - ŷ_i)^{2}}$$$$L = 1/2 * 1/n * \sum{(y_i - (w_j * x_j + b_j))^{2}}$$$$x_j = relu(w_i * x_i + b_i)$$**Gradients**$$\frac{\partial L}{\partial w_i} = 1/n * \sum{(y_i - ŷ_i)} * x_j $$$$\frac{\partial L}{\partial b_i} = 1/n * \sum{(y_i - ŷ_i)} * 1$$$$\frac{\partial L}{\partial w_j} = 1/n * \sum{(y_i - ŷ_i)} * x_j * x_i, se relu() > 0$$$$\frac{\partial L}{\partial b_j} = 1/n * \sum{(y_i - ŷ_i)} * x_j, se relu() > 0$$  Treinando<jupyter_code>nn = NeuralNetwork()
t = Trainer(nn, verbose=False)
for i in range(100000):
    t.train(synt_x, synt_y)

t.plot_losses()<jupyter_output><empty_output><jupyter_text>Comparando com a realidade<jupyter_code>print('Parâmetros aprendidos:')
print('pesos:', nn.weights)
print('bias:', nn.biases)
print('Função que modela os dados: 7 * X + 15')
plot_line(synt_x, nn.run_batch(synt_x), '--r')
plot_line(synt_x, synt_y)
plt.show()<jupyter_output>Parâmetros aprendidos:
pesos: [array([[6.69336918]])]
bias: [array([15.07485933])]
Função que modela os dados: 7 * X + 15<jupyter_text>Uma função um pouco mais complicada$Y = 7 * log(x) + 1$<jupyter_code>def get_random_error(size, mu=0, std_dev=0.8):
    return np.random.normal(mu, std_dev, size)

synt_x = np.random.rand(SYNT_TRAIN_SIZE)
synt_y = np.reshape(7 * np.log(synt_x) + 1 + get_random_error(SYNT_TRAIN_SIZE), (SYNT_TRAIN_SIZE, 1))

synt_x = np.reshape(synt_x, (SYNT_TRAIN_SIZE, 1))
plt.plot(synt_x, synt_y, 'ro', alpha=0.5)
nn = NeuralNetwork(layers=[10, 1], activations=['sigmoid', None])
t = Trainer(nn)
for i in range(10000):
    t.train(synt_x, synt_y)

t.plot_losses()
print('Parâmetros aprendidos:')
print('pesos:', nn.weights)
print('bias:', nn.biases)
print('Função que modela os dados: 7 * X + 15')
plt.plot(synt_x, nn.run_batch(synt_x), 'or', alpha=0.3)
plt.plot(synt_x, synt_y, 'og', alpha=0.3)
plt.show()<jupyter_output>Parâmetros aprendidos:
pesos: [array([[ -5.36855323,  -3.03145738,  -5.21423527, -14.09143894,
         -0.83027733,  -2.08127383,  -1.48456986,   2.56702234,
         -3.32490432,   4.39467954]]), array([[ -6.11528773],
       [ -3.83950468],
       [ -6.11003906],
       [-14.98979137],
       [ -2.35375611],
       [ -2.63361214],
       [ -2.25129261],
       [  1.55817272],
       [ -1.84865704],
       [  4.95015308]])]
bias: [array([ 0.47915267, -0.06928394, -0.20321051,  0.77060138,  2.57464764,
       -1.61306825, -0.24671431,  0.04661517, -2.91871913, -2.46777321]), array([-2.53449749])]
Função que modela os dados: 7 * X + 15<jupyter_text>E se os dados forem não lineares?<jupyter_code>xor_x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
xor_y = np.array([[0], [1], [1], [0]])
nn = NeuralNetwork(layers=[10, 2], input_size=2, activations=['relu', None])
t = Trainer(nn, verbose=False)
for i in range(100000):
    t.train(xor_x, xor_y)

t.plot_losses()
plt.plot(xor_x, nn.run_batch(xor_x), 'bo', xor_x, xor_y, 'ro', alpha=0.3)
nn = NeuralNetwork(layers=[10, 1], input_size=2, activations=[None, None])
t = Trainer(nn, verbose=False)
for i in range(100000):
    t.train(xor_x, xor_y)

t.plot_losses()
plt.plot(xor_x, nn.run_batch(xor_x), 'bo', xor_x, xor_y, 'ro', alpha=0.3)<jupyter_output><empty_output><jupyter_text>Exemplo: [base dados Iris](https://mari-linhares.github.io/codando-deep-learning/notebooks/glossario.htmliris)Digamos que para um exemplo da base de dados queremos determinar qual a espécie dessa planta.**Entradas**A base de dados iris tem 4 **[atributos](https://mari-linhares.github.io/codando-deep-learning/notebooks/glossario.htmlatributos)** de uma planta que iremos usar como entrada.**Saídas**Neste caso a saída que nos interessa é a espécie da planta. Então digamos que a saída é um número que indica qual a espécie:0 = Iris Setosa , 1 = Iris Versicolour, 2 = Iris Virginica  Obtendo os dados<jupyter_code># Carregar base de dados iris
iris = fetch_mldata('iris')

# np.c_ concatena as features e targets do dataset
iris_data = pd.DataFrame(data=np.c_[iris['data'], iris['target']],
                         columns=['x0', 'x1', 'x2', 'x3', 'target'])
iris_data.head()
iris_data.describe()
iris_data.drop(['target'], axis=1).diff().hist(color='k', alpha=0.5, bins=10, figsize=(4, 5))
plt.show()
x = iris.data
y = iris.target
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)
def batches(x, y, batch_size=True):
    idx = np.random.permutation(len(x))
    x = x[idx]
    y = y[idx]
    
    for i in range(0, len(x)-batch_size-1, batch_size):
        batch_x = x[i:i+batch_size]
        batch_y = y[i:i+batch_size]
        yield batch_x, batch_y
nn = NeuralNetwork(layers=[10, 4], input_size=4, activations=['relu', None])
t = Trainer(nn, verbose=False, loss_name='cross-entropy')
for i in range(1000):
    for batch_x, batch_y in batches(x, y, 16):
        t.train(batch_x, batch_y)
    if i % 10 == 0:
        loss, metrics = t.eval(x_test, y_test, metrics=['accuracy'])
        print('Test loss = %.5f, accuracy %.5f' % (loss, metrics[0]))

t.plot_losses()<jupyter_output>Test loss = 1.21398, accuracy 0.44737
Test loss = 1.19670, accuracy 0.52632
Test loss = 1.20644, accuracy 0.55263
Test loss = 1.20227, accuracy 0.55263
Test loss = 1.17978, accuracy 0.55263
Test loss = 1.12980, accuracy 0.55263
Test loss = 1.04272, accuracy 0.52632
Test loss = 0.90289, accuracy 0.63158
Test loss = 0.72344, accuracy 0.94737
Test loss = 0.57425, accuracy 0.94737
Test loss = 0.48888, accuracy 0.94737
Test loss = 0.44399, accuracy 0.94737
Test loss = 0.42637, accuracy 0.94737
Test loss = 0.41667, accuracy 0.94737
Test loss = 0.40954, accuracy 0.94737
Test loss = 0.40582, accuracy 0.94737
Test loss = 0.40292, accuracy 0.94737
Test loss = 0.39989, accuracy 0.94737
Test loss = 0.40108, accuracy 0.94737
Test loss = 0.40485, accuracy 0.92105
Test loss = 0.40582, accuracy 0.92105
Test loss = 0.40479, accuracy 0.92105
Test loss = 0.40645, accuracy 0.92105
Test loss = 0.40586, accuracy 0.92105
Test loss = 0.40694, accuracy 0.92105
Test loss = 0.40653, accuracy 0.92105
Test loss = [...]<jupyter_text>MNIIST<jupyter_code># load mnist
mnist = fetch_mldata('MNIST original')
x = mnist.data / np.max(mnist.data)
y = mnist.target
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)
nn = NeuralNetwork(layers=[512, 256, 10], input_size=784, activations=['relu', 'relu', None])
t = Trainer(nn, verbose=False, loss_name='cross-entropy', learning_rate=0.001)
for i in range(20):
    for batch_x, batch_y in batches(x, y, 64):
        t.train(batch_x, batch_y)
    if i % 1 == 0:
        loss, metrics = t.eval(x_test, y_test, metrics=['accuracy'])
        print('Test loss = %.5f, accuracy %.5f' % (loss, metrics[0]))

t.plot_losses()<jupyter_output><empty_output><jupyter_text>Em busca da função perfeita Exemplo: [base dados Iris](https://mari-linhares.github.io/codando-deep-learning/notebooks/glossario.htmliris)Digamos que para um exemplo da base de dados queremos determinar qual a espécie dessa planta.**Entradas**A base de dados iris tem 4 **[atributos](https://mari-linhares.github.io/codando-deep-learning/notebooks/glossario.htmlatributos)** de uma planta que iremos usar como entrada.**Saídas**Neste caso a saída que nos interessa é a espécie da planta. Então digamos que a saída é um número que indica qual a espécie:0 = Iris Setosa , 1 = Iris Versicolour, 2 = Iris Virginica Obtendo a base de dados<jupyter_code># Ignorar warnings
import warnings
warnings.filterwarnings('ignore')

# Trabalhar com os dados
from sklearn.datasets import fetch_mldata

# Atributos: iris_dataset.data
# Espécie: iris_dataset.target
iris_dataset = fetch_mldata('iris')

print('Numero de exemplos na base:', len(iris_dataset.data))
print('Atributos da primeira planta:', iris_dataset.data[0])
print('Especie da primeira planta:', iris_dataset.target[0])<jupyter_output><empty_output><jupyter_text>Vamos codar uma função que resolve esse problema!Uma função para resolver esse problema precisa receber 4 parâmetros (cada um dos atributos da planta) e produzir uma saída (espécie da planta).<jupyter_code>def f(x1, x2, x3, x4):
    # Não importa os atributos pra mim a resposta é sempre: Setosa!!!!
    return 0

print(f(*iris_dataset.data[0]), iris_dataset.target[0])
print(f(*iris_dataset.data[1]), iris_dataset.target[1])
print(f(*iris_dataset.data[-1]), iris_dataset.target[-1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1400499229911307, 'fraction_numerical': 0.04843592330978809, 'mean_word_length': 2.9658803706823926, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 50, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '136300', 'license': 'Apache-2.0', 'n_tokens_mistral': 7668, 'n_tokens_neox': 7061, 'n_words': 1575, 'path': '_notebooks/2021-01-21-1.5.minha_primeira_rede_neural.ipynb', 'repo_name': 'hereismari/codando-redes-neurais', 'size': '18829'}