<jupyter_start><jupyter_text>Neuronale Netze   Neuronen  Künstliche Neuronen  Künstliche Neuronen  Aktivierungsfunktionen<jupyter_code>import torch
import torch.nn as nn
import numpy as np
import matplotlib.pyplot as plt
act_x = torch.tensor(np.linspace(-6, 6, 100))
plt.figure(figsize=(16, 12))
plt.subplot(3, 2, 1)
plt.plot(act_x, nn.Sigmoid()(act_x))
plt.subplot(3, 2, 2)
plt.plot(act_x, nn.Tanh()(act_x))
plt.subplot(3, 2, 3)
plt.plot(act_x, nn.ReLU()(act_x))
plt.subplot(3, 2, 4)
plt.plot(act_x, - nn.ReLU()(act_x + 2))
plt.subplot(3, 2, 5)
plt.plot(act_x, nn.ReLU()(act_x) - nn.ReLU()(act_x + 2))
plt.subplot(3, 2, 6)
plt.plot(act_x, nn.Tanh()(act_x) - 1.5 * nn.Tanh()(act_x - 2))
import torch
import torch.nn as nn


neuron = lambda x: nn.Tanh()(nn.Linear(4, 1)(x))
neuron(torch.tensor([1.0, 2.0, 3.0, 4.0]))
neuron = nn.Sequential(
    nn.Linear(4, 1),
    nn.Tanh()
)
neuron(torch.tensor([1.0, 2.0, 3.0, 4.0]))<jupyter_output><empty_output><jupyter_text>Neuronale Netze<jupyter_code>seq_model = nn.Sequential(
    nn.Linear(2, 4),
    nn.ReLU(),
    nn.Linear(4, 3),
    nn.ReLU(),
    nn.Linear(3, 2)
)
seq_model(torch.tensor([1.0, 2.0]))<jupyter_output><empty_output><jupyter_text>Erinnerung: Training  Training Neuraler Netze  Training Neuraler Netze  Training Neuraler Netze  Training Neuraler Netze  Training Neuraler Netze  Wie updaten wir die Parameter?  Wie updaten wir die Parameter?  Wie updaten wir die Parameter?  MNIST<jupyter_code>import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn

input_size = 28 * 28
num_classes = 10
num_epochs = 5
batch_size = 100
learning_rate = 0.005
mnist_transforms = transforms.Compose([
    transforms.Resize(28, 28),
    transforms.ToTensor()
])
train_dataset = torchvision.datasets.MNIST(root='./data',
                                           train=True,
                                           transform=mnist_transforms,
                                           download=True)
test_dataset = torchvision.datasets.MNIST(root='./data',
                                          train=False,
                                          transform=mnist_transforms,
                                          download=True)
it = iter(train_dataset)
next(it)[0].shape, next(it)[1]
next(it)[0].shape, next(it)[1]
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)
def create_model(hidden_size):
    model = nn.Sequential(
        nn.Linear(input_size, hidden_size),
        nn.ReLU(),
        nn.Linear(hidden_size, num_classes)
    )
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    return model, optimizer
loss_fn = nn.CrossEntropyLoss()
m = torch.randn(2, 3, 2, 5)
m.reshape(-1, 30).shape
def training_loop(n_epochs, optimizer, model, loss_fn, device, train_loader, val_loader):
    all_losses = []
    for epoch in range(1, n_epochs + 1):
        accumulated_loss = 0
        for i, (images, labels) in enumerate(train_loader):
            images = images.reshape(-1, input_size).to(device)
            labels = labels.to(device)

            output = model(images)
            loss = loss_fn(output, labels)
            with torch.no_grad():
                accumulated_loss += loss
                all_losses.append(loss)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        
            if (i + 1) % 100 == 0:
                print(f"Epoch {epoch:3}/{n_epochs:3}, step {i + 1}: "
                      f"training loss = {accumulated_loss.item():8.3f}")
                accumulated_loss = 0
    return all_losses
def run_model(hidden_size, num_epochs=num_epochs):
    model, optimizer = create_model(hidden_size)
    losses = training_loop(
        n_epochs=num_epochs,
        optimizer=optimizer,
        model=model,
        loss_fn=loss_fn,
        device=torch.device('cpu') if torch.cuda.is_available() else torch.device('cpu'),
        train_loader=train_loader,
        val_loader=test_loader
    )
    return losses
losses = run_model(128, num_epochs=5)
from matplotlib import pyplot
pyplot.figure(figsize=(16, 5))
pyplot.plot(range(len(losses)), losses);
run_model(32)
from matplotlib import pyplot
pyplot.figure(figsize=(16, 5))
pyplot.plot(range(len(losses)), losses);
run_model(512, num_epochs=10)
from matplotlib import pyplot
pyplot.figure(figsize=(16, 5))
pyplot.plot(range(len(losses)), losses);<jupyter_output>Epoch   1/ 10, step 100: training loss =   42.237
Epoch   1/ 10, step 200: training loss =   19.806
Epoch   1/ 10, step 300: training loss =   15.659
Epoch   1/ 10, step 400: training loss =   14.877
Epoch   1/ 10, step 500: training loss =   13.241
Epoch   1/ 10, step 600: training loss =   12.211
Epoch   2/ 10, step 100: training loss =    8.249
Epoch   2/ 10, step 200: training loss =    8.177
Epoch   2/ 10, step 300: training loss =    9.035
Epoch   2/ 10, step 400: training loss =    8.776
Epoch   2/ 10, step 500: training loss =    8.792
Epoch   2/ 10, step 600: training loss =    8.950
Epoch   3/ 10, step 100: training loss =    6.073
Epoch   3/ 10, step 200: training loss =    5.804
Epoch   3/ 10, step 300: training loss =    6.615
Epoch   3/ 10, step 400: training loss =    6.450
Epoch   3/ 10, step 500: training loss =    6.391
Epoch   3/ 10, step 600: training loss =    6.566
Epoch   4/ 10, step 100: training loss =    4.324
Epoch   4/ 10, step 200: training loss =    4.841
[...]<jupyter_text>Modelle  Für Neuronale Netze:Was repräsentiert werden kann hängt ab von- Anzahl der Layers- Anzahl der Neutronen per Layer- Komplexität der Verbindungen zwischen Neutronen  Was kann man (theoretisch) lernen?Schwierig aber irrelevant  Was kann man praktisch lernen?Sehr viel, wenn man genug Zeit und Daten hat  Was kann man effizient lernen?Sehr viel, wenn man sich geschickt anstellt(und ein Problem hat, an dem viele andere Leute arbeiten)  Bias/Variance Tradeoff- Modelle mit geringer Expressivität (representational power)  - Können schnell trainiert werden  - Arbeiten mit wenig Trainingsdaten  - Sind robust gegenüber Fehlern in den Trainingsdaten- Wir sind nicht an einer möglichst exakten Wiedergabe unserer Daten interessiert- Entscheidend ist wie gut unser Modell auf unbekannte Daten generalisiert      Generalisierung und Rauschen    Komplexität der Entscheidungsgrenze   Datenverteilung und Qualität  Erinnerung: die Trainings-Schleife   Was lernt ein Klassifizierer?    Wie gut sind wir?Wie wissen wir, wie gut unser Modell wirklich ist?  Was kann schief gehen?  Was kann schief gehen?  Was kann schief gehen?  Accuracy: Wie viel haben wir richtig gemacht?  Precision: Wie gut sind unsere positiven Elemente?  Recall: Wie viele positive Elemente haben wir übersehen?  Bessere Netzwerkarchitektur    Beispiel: Conv Net<jupyter_code>def create_conv_model():
    model = nn.Sequential(
        nn.Conv2d(1, 32, 3, 1),
        nn.ReLU(),
        nn.Conv2d(32, 64, 3, 1),
        nn.MaxPool2d(2),
        nn.Dropout2d(0.25),
        nn.Flatten(1),
        nn.Linear(9216, 128),
        nn.ReLU(),
        nn.Dropout2d(0.5),
        nn.Linear(128, 10)
    )
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    return model, optimizer
def training_loop(n_epochs, optimizer, model, loss_fn, device, train_loader, val_loader):
    all_losses = []
    for epoch in range(1, n_epochs + 1):
        accumulated_loss = 0
        for i, (images, labels) in enumerate(train_loader):
            images = images.to(device)
            labels = labels.to(device)

            output = model(images)
            loss = loss_fn(output, labels)
            with torch.no_grad():
                accumulated_loss += loss
                all_losses.append(loss)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        
            if (i + 1) % 100 == 0:
                print(f"Epoch {epoch:3}/{n_epochs:3}, step {i + 1}: "
                      f"training loss = {accumulated_loss.item():8.3f}")
                accumulated_loss = 0
    return all_losses
def run_conv_model(num_epochs=num_epochs):
    model, optimizer = create_conv_model()
    losses = training_loop(
        n_epochs=num_epochs,
        optimizer=optimizer,
        model=model,
        loss_fn=loss_fn,
        device=torch.device('cpu') if torch.cuda.is_available() else torch.device('cpu'),
        train_loader=train_loader,
        val_loader=test_loader
    )
run_conv_model(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1082115518441197, 'fraction_numerical': 0.04442124797030851, 'mean_word_length': 2.8824853669518236, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '426351', 'license': 'Apache-2.0', 'n_tokens_mistral': 3240, 'n_tokens_neox': 2955, 'n_words': 751, 'path': '020 Neuronale Netze.ipynb', 'repo_name': 'hoelzl/advanced-ml-course', 'size': '8622'}