<jupyter_start><jupyter_text>Exam Preparation Deep Learning18.03.2022, <NAME>  2. AutoEncoder  2.1 Initialization<jupyter_code>import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
from torchvision.utils import save_image

#
# Setup device (cuda / cpu)
#
cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
print("Device:")
print(device)<jupyter_output>Device:
cpu<jupyter_text>2.2 Download data<jupyter_code>train_data = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor())                       # ToTensor() erstellt proprietäres Format für PyTorch

test_data = datasets.FashionMNIST(
    root="data",
    train=False,
    download=True,
    transform=ToTensor())<jupyter_output><empty_output><jupyter_text>2.3 Explore data<jupyter_code>print("#")
print("# Train Data")
print("#")
print(train_data)

print("\n#")
print("# Train Data Classes")
print("#")
print(train_data.classes)
print()

print("\n#")
print("# Train Data ClassToIndex")
print("#")
print(train_data.class_to_idx)<jupyter_output>#
# Train Data
#
Dataset FashionMNIST
    Number of datapoints: 60000
    Root location: data
    Split: Train
    StandardTransform
Transform: ToTensor()

#
# Train Data Classes
#
['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']


#
# Train Data ClassToIndex
#
{'T-shirt/top': 0, 'Trouser': 1, 'Pullover': 2, 'Dress': 3, 'Coat': 4, 'Sandal': 5, 'Shirt': 6, 'Sneaker': 7, 'Bag': 8, 'Ankle boot': 9}<jupyter_text>2.4 Create Data Loader<jupyter_code>batch_size = 64
train_dataloader = DataLoader(train_data, batch_size=batch_size)
test_dataloader  = DataLoader(test_data, batch_size=batch_size, shuffle=True)

for X, y in train_dataloader:
    print("Shape of X [N, C, H, W]: ", X.shape)
    print("Shape of y: ", y.shape, y.dtype)
    break<jupyter_output>Shape of X [N, C, H, W]:  torch.Size([64, 1, 28, 28])
Shape of y:  torch.Size([64]) torch.int64<jupyter_text>2.5 AutoEncoder<jupyter_code>class AutoEncoder(torch.nn.Module):
    """AutoEncoder with:  28*28 (784) >> 9 >> 28*28"""

    def __init__(self, n_features=28 * 28):
        super().__init__()

        # Building an linear encoder with Linear layer followed by Relu activation function: 28*28 >> 9
        self.encoder = torch.nn.Sequential(
            torch.nn.Linear(n_features, 128),
            torch.nn.ReLU(),
            torch.nn.Linear(128, 64),
            torch.nn.ReLU(),
            torch.nn.Linear(64, 36),
            torch.nn.ReLU(),
            torch.nn.Linear(36, 18),
            torch.nn.ReLU(),
            torch.nn.Linear(18, 9)
        )

        # Building an linear decoder with Linear layer followed by Relu activation function: 9 >> 28*28
        # The Sigmoid activation function outputs the value between 0 and 1
        self.decoder = torch.nn.Sequential(
            torch.nn.Linear(9, 18),
            torch.nn.ReLU(),
            torch.nn.Linear(18, 36),
            torch.nn.ReLU(),
            torch.nn.Linear(36, 64),
            torch.nn.ReLU(),
            torch.nn.Linear(64, 128),
            torch.nn.ReLU(),
            torch.nn.Linear(128, n_features),
            torch.nn.Sigmoid()
        )

    def forward(self, x):
        encoded = self.encoder(x)
        decoded = self.decoder(encoded)
        return decoded

model = AutoEncoder().to(device)                  # Angabe wo das ausgeführt werden soll
print(model)<jupyter_output>AutoEncoder(
  (encoder): Sequential(
    (0): Linear(in_features=784, out_features=128, bias=True)
    (1): ReLU()
    (2): Linear(in_features=128, out_features=64, bias=True)
    (3): ReLU()
    (4): Linear(in_features=64, out_features=36, bias=True)
    (5): ReLU()
    (6): Linear(in_features=36, out_features=18, bias=True)
    (7): ReLU()
    (8): Linear(in_features=18, out_features=9, bias=True)
  )
  (decoder): Sequential(
    (0): Linear(in_features=9, out_features=18, bias=True)
    (1): ReLU()
    (2): Linear(in_features=18, out_features=36, bias=True)
    (3): ReLU()
    (4): Linear(in_features=36, out_features=64, bias=True)
    (5): ReLU()
    (6): Linear(in_features=64, out_features=128, bias=True)
    (7): ReLU()
    (8): Linear(in_features=128, out_features=784, bias=True)
    (9): Sigmoid()
  )
)<jupyter_text>2.6 Loss und Optimizer<jupyter_code># Backward
# - Definition Backward Loss Funktion
# - Build partial derivates of loss function and grads
loss_fn = nn.MSELoss()

# Step
# - Optimizer aktualisiert die Gewichte
# - Der optimizer macht den Update step intelligenter
# - Update: params = params - learing_rate * grad
# - Besser ist Adam optimizer
#   optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)<jupyter_output><empty_output><jupyter_text>2.7 Train und Test Functions<jupyter_code>def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    model.train()

    for batch, (X, y) in enumerate(dataloader):     # Mit enumerate erhält man batch nummer
        #
        # Reshape
        #
        X = X.reshape(-1, 28 * 28)                  # reshape n_features 28*28
        X = X.to(device)

        #
        # Compute prediction error
        #
        pred = model(X)                             # Impliziter Aufruf von model.forward
        loss = loss_fn(pred, X)

        #
        # Backpropagation
        #
        optimizer.zero_grad()                       # Reset Anfangswerte für jeden Batch

        loss.backward()                             # X.size Ableitungen und Werte
                                                    # 1. Partielle Ableitungen bilden
                                                    # 2. Werte werden in Ableitungsformel ein geplugged
                                                    #    X Werte, Y Werte, Gewichte und Bias
                                                    #    Pro Ableitung erhlät man einen Skalar (m_deriv)
                                                    #    Ein Gradient pro Feature und Bias

        optimizer.step()                            # Update der Gradienten (konkrete Werte der Ableitung)
                                                    # Gewichte/Bias = Gewichte/Bias bisher - LR * Gradienten

        #
        # Zwischenresultate zeigen
        #
        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X) # batch * len(X) = Position Build
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")


def test(dataloader, model, loss_fn):
    num_batches = len(dataloader)
    model.eval()
    test_loss = 0

    with torch.no_grad():                           # no_grad: keine Aufzeichnung, nur Prediction
        for X, y in dataloader:
            X = X.reshape(-1, 28 * 28)              # reshape n_features 28*28
            X = X.to(device)

            pred = model(X)
            test_loss += loss_fn(pred, X).item()    # item() holt den Wert aus dem Objekt

        test_loss /= num_batches
        print(f"Test Error: \n Avg loss: {test_loss:>8f} \n")<jupyter_output><empty_output><jupyter_text>2.8 Training<jupyter_code>epochs = 10 # 100

for epoch in range(epochs):
    print("Epoch:", epoch+1, "\n-------------------------------")
    train(dataloader=train_dataloader, model=model, loss_fn=loss_fn, optimizer=optimizer)
    test( dataloader=test_dataloader,  model=model, loss_fn=loss_fn)
    # save weigths
    # torch.save(model.state_dict(), filename=f"model_epoch_{epoch}.pth")<jupyter_output>Epoch: 1 
-------------------------------
loss: 0.171169  [    0/60000]
loss: 0.063880  [ 6400/60000]
loss: 0.057043  [12800/60000]
loss: 0.040360  [19200/60000]
loss: 0.041132  [25600/60000]
loss: 0.037380  [32000/60000]
loss: 0.030040  [38400/60000]
loss: 0.030992  [44800/60000]
loss: 0.030865  [51200/60000]
loss: 0.029687  [57600/60000]
Test Error: 
 Avg loss: 0.031179 

Epoch: 2 
-------------------------------
loss: 0.030771  [    0/60000]
loss: 0.027877  [ 6400/60000]
loss: 0.027732  [12800/60000]
loss: 0.026731  [19200/60000]
loss: 0.028001  [25600/60000]
loss: 0.029044  [32000/60000]
loss: 0.025225  [38400/60000]
loss: 0.025744  [44800/60000]
loss: 0.026972  [51200/60000]
loss: 0.025182  [57600/60000]
Test Error: 
 Avg loss: 0.026938 

Epoch: 3 
-------------------------------
loss: 0.027602  [    0/60000]
loss: 0.026073  [ 6400/60000]
loss: 0.024830  [12800/60000]
loss: 0.026350  [19200/60000]
loss: 0.025740  [25600/60000]
loss: 0.026849  [32000/60000]
loss: 0.023861  [38400/6[...]<jupyter_text>2.9 Test images with noise<jupyter_code>path='autoencoder/'

def test_with_testimage(model, test_image, epoch, filename):
    model.eval()
    with torch.no_grad():
        test_image = test_image.reshape(-1, 28 * 28)
        test_image = test_image.to(device)
        predicted_image = model(test_image)
        predicted_image = predicted_image.reshape(-1, 28, 28)
        fn = path + f"{filename}-epoch-{epoch}.png"
        save_image(predicted_image[0], fn)

test_image = test_data.data[0, :, :].float()
save_image(test_image, path + "image.png")

noise = torch.randn(test_image.shape) * 0.1                     # Add more noise with 0.2, 0.3...
noisy_test_image = torch.add(test_image, noise)
save_image(noisy_test_image, path + "image-noisy.png")

epochs = 10 # 100
for epoch in range(epochs):
    print("Epoch:", epoch+1, "\n-------------------------------")
    train(dataloader=train_dataloader, model=model, loss_fn=loss_fn, optimizer=optimizer)
    test_with_testimage(model=model, test_image=test_image, epoch=epoch, filename="test")
    test_with_testimage(model=model, test_image=noisy_test_image, epoch=epoch, filename="test-noisy")
    # save weigths
    # torch.save(model.state_dict(), filename=f"model_epoch_{epoch}.pth")<jupyter_output>Epoch: 1 
-------------------------------
loss: 0.019624  [    0/60000]
loss: 0.019065  [ 6400/60000]
loss: 0.017773  [12800/60000]
loss: 0.019153  [19200/60000]
loss: 0.019114  [25600/60000]
loss: 0.020253  [32000/60000]
loss: 0.018500  [38400/60000]
loss: 0.016867  [44800/60000]
loss: 0.018842  [51200/60000]
loss: 0.019182  [57600/60000]
Epoch: 2 
-------------------------------
loss: 0.019294  [    0/60000]
loss: 0.018893  [ 6400/60000]
loss: 0.017339  [12800/60000]
loss: 0.019122  [19200/60000]
loss: 0.018663  [25600/60000]
loss: 0.020012  [32000/60000]
loss: 0.018484  [38400/60000]
loss: 0.016574  [44800/60000]
loss: 0.018603  [51200/60000]
loss: 0.018978  [57600/60000]
Epoch: 3 
-------------------------------
loss: 0.019105  [    0/60000]
loss: 0.018714  [ 6400/60000]
loss: 0.016996  [12800/60000]
loss: 0.018815  [19200/60000]
loss: 0.018600  [25600/60000]
loss: 0.019764  [32000/60000]
loss: 0.018125  [38400/60000]
loss: 0.016358  [44800/60000]
loss: 0.018792  [51200/60000]
loss[...]
{'fraction_non_alphanumeric': 0.15297995056303212, 'fraction_numerical': 0.10848667948365834, 'mean_word_length': 2.561786762308445, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '414207', 'license': 'MIT', 'n_tokens_mistral': 4689, 'n_tokens_neox': 3926, 'n_words': 853, 'path': 'src/20_exam/exam-dl2-auto-encoder.ipynb', 'repo_name': 'surfmachine/ai', 'size': '10928'}