<jupyter_start><jupyter_text>AMAL - TP 4 - Réseaux récurrents Bonjour, comme indiqué ce matin, le code ne fonctionne pas. Je l'uploade afin que vous puissez faire un retour et que je comprenne les erreurs. Je le complèterai ensuite. En particulier, comme je n'arrive pas à faire diminuer l'erreur d'apprentissage, je n'ai pas encore séparé le dataset en train / validation / test. Merci !<jupyter_code>import numpy as np
import torch
from datamaestro import prepare_dataset 
from torch.utils.tensorboard import SummaryWriter
import time
import csv
import pandas as pd
import os
from matplotlib import pyplot as plt
torch.autograd.set_detect_anomaly(True)<jupyter_output><empty_output><jupyter_text>Exercice 1<jupyter_code>class RNN(torch.nn.Module):
    def __init__(self, dim, latent, batch):
        super(RNN, self).__init__()
        
        self.Wi = torch.nn.Linear(dim, latent, bias=False)
        self.Wh = torch.nn.Linear(latent, latent)
        self.latent = latent
        self.h = torch.zeros(batch, latent, requires_grad=True)
        
    def one_step(self, x, h):
        return torch.nn.Tanh()(self.Wi(x) + self.Wh(h)) 
    
    def forward(self, x):
        length, batch = x.shape[0], x.shape[1] 
        hs = []
        hs.append(self.one_step(x[0], self.h))
        for i in range(1, length):
            hs.append(self.one_step(x[i], hs[-1]))
        return hs[-1]    # pour cet exercice, hs[-1] suffit. Je généraliserai pour les exercices suivants.
       # return torch.cat(hs, dim=0)<jupyter_output><empty_output><jupyter_text>Je fais juste le test avec 2 villes pour l'instant<jupyter_code>n_cities = 2
dim = 1
batch = 16
latent = 20
length = 100
temp_data = pd.read_csv('tempAMAL_train.csv')
temp_data = temp_data.iloc[:,1: 1+n_cities]
data = np.asarray(temp_data)
cols = temp_data.columns

n_times = data.shape[0]
data[np.isnan(data)] = np.mean(data[~np.isnan(data)])
data = (data - np.min(data)) / (np.max(data) - np.min(data))
increment = np.repeat(np.arange(0, length), batch).reshape((length, batch))
model = torch.nn.Sequential(
        RNN(dim, latent, batch),
        torch.nn.Linear(latent, n_cities)
   #     torch.nn.Sigmoid() 
)
def test_model(n_test):
    p_true = 0

    for i in range(n_test):
        with torch.no_grad():
            random_cities = np.random.randint(0, n_cities, size=(batch))
            random_times = np.random.randint(0, n_times-length-dim, size=(batch))
            x_numpy = np.zeros((length, batch, dim))
            for k in range(dim):
                x_numpy[:, :, k] = data[np.repeat(np.reshape(random_times+k, (1, batch)), length, axis=0)+increment, random_cities]
            x = torch.from_numpy(x_numpy).float()
            predicted_cities = model(x).detach().numpy()
            ok = np.sum(np.argmax(predicted_cities, axis=1) == random_cities) / batch
            p_true += ok

    print(p_true / n_test)
    
test_model(100)
#optim = torch.optim.SGD(params=model.parameters(),lr=EPS)
optim = torch.optim.Adam(params=[x for x in model.parameters()] + [model[0].h], lr=1e-3,betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)
loss = torch.nn.CrossEntropyLoss()
n_train = 100
batch * length * dim * n_train / (data.shape[0]*data.shape[1])
writer = SummaryWriter()
losses = []
for i in range(n_train):
    # each time: we see batch * length data
    optim.zero_grad()
    random_cities = np.random.randint(0, n_cities, size=(batch))
    random_times = np.random.randint(0, n_times-length-dim, size=(batch))
    x_numpy = np.zeros((length, batch, dim))
    for k in range(dim):
        x_numpy[:, :, k] = data[np.repeat(np.reshape(random_times+k, (1, batch)), length, axis=0)+increment, random_cities]
    x = torch.from_numpy(x_numpy).float()
    l = loss(model(x), torch.from_numpy(random_cities))
    l.backward()
    optim.step()
    writer.add_scalar('Loss/train', l, i)
    losses.append(l)
plt.plot(np.cumsum(losses))
print(model[0].Wi.weight.grad)
print(model[0].Wi.weight)
test_model(100)<jupyter_output>0.515625
{'fraction_non_alphanumeric': 0.14025455452957325, 'fraction_numerical': 0.01672073870726229, 'mean_word_length': 3.5339366515837103, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '400990', 'license': 'MIT', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1486, 'n_words': 324, 'path': 'AMAL/week4/td4.ipynb', 'repo_name': 'PierreMarion23/ML_exercises', 'size': '4007'}