<jupyter_start><jupyter_text>Generative Adversarial NetworksThis code is based on https://arxiv.org/abs/1406.2661 paper from <NAME>, <NAME>, et all ![title](https://github.com/DSC-UI-SRIN/Introduction-to-GAN/raw/master/1%20-%20Fundamental%20of%20GANs/images/minmax.png)<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# import All prerequisites
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from torchvision.utils import save_image
import numpy as np

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
ROOT = "/content/drive/My Drive/Colab Notebooks/DSC_UI_GAN/Batch1/W1/"<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>batch_size = 100

# MNIST Dataset
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.5], [0.5])])

train_dataset = datasets.MNIST(root='./mnist_data/', train=True, transform=transform, download=True)

# Data Loader (Input Pipeline)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)

examples = enumerate(train_loader)
batch_idx, (example_data, example_targets) = next(examples)
## Print example
import matplotlib.pyplot as plt

fig = plt.figure()
for i in range(6):
  plt.subplot(2,3,i+1)
  plt.tight_layout()
  plt.imshow(example_data[i][0], cmap='gray', interpolation='none')
  plt.title("Ground Truth: {}".format(example_targets[i]))
  plt.xticks([])
  plt.yticks([])
fig<jupyter_output><empty_output><jupyter_text>Build Network ![title](https://miro.medium.com/max/1600/1*M_YipQF_oC6owsU1VVrfhg.jpeg)resource : https://medium.com/@jonathan_hui/gan-whats-generative-adversarial-networks-and-its-application-f39ed278ef09<jupyter_code>class Discriminator(nn.Module):
    def __init__(self, d_input_dim):
        super(Discriminator, self).__init__()

        self.model = nn.Sequential(
            nn.Linear(d_input_dim, 512),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(512, 256),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Linear(256, 1),
            nn.Sigmoid(),
        )

    def forward(self, image):
        img_flat = image.view(image.size(0), -1)
        validity = self.model(img_flat)
        return validity
    
class Generator(nn.Module):
    def __init__(self, g_input_dim, g_output_dim):
        super(Generator, self).__init__()

        def block(in_feat, out_feat, normalize=True):
            layers = [nn.Linear(in_feat, out_feat)]
            if normalize:
                layers.append(nn.BatchNorm1d(out_feat, 0.8))
            layers.append(nn.LeakyReLU(0.2, inplace=True))
            return layers

        self.model = nn.Sequential(
            *block(g_input_dim, 128, normalize=False),
            *block(128, 256),
            *block(256, 512),
            *block(512, 1024),
            nn.Linear(1024, g_output_dim), 
            nn.Tanh())

    def forward(self, z):
        image = self.model(z)
        image = image.view(image.size(0), -1)
        return image
# build network
z_dim = 100
mnist_dim = train_dataset.train_data.size(1) * train_dataset.train_data.size(2)

G = Generator(g_input_dim = z_dim, g_output_dim = mnist_dim).to(device)
D = Discriminator(mnist_dim).to(device)
print(G, D)<jupyter_output>Generator(
  (model): Sequential(
    (0): Linear(in_features=100, out_features=128, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace=True)
    (2): Linear(in_features=128, out_features=256, bias=True)
    (3): BatchNorm1d(256, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (4): LeakyReLU(negative_slope=0.2, inplace=True)
    (5): Linear(in_features=256, out_features=512, bias=True)
    (6): BatchNorm1d(512, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (7): LeakyReLU(negative_slope=0.2, inplace=True)
    (8): Linear(in_features=512, out_features=1024, bias=True)
    (9): BatchNorm1d(1024, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (10): LeakyReLU(negative_slope=0.2, inplace=True)
    (11): Linear(in_features=1024, out_features=784, bias=True)
    (12): Tanh()
  )
) Discriminator(
  (model): Sequential(
    (0): Linear(in_features=784, out_features=512, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace=True)[...]<jupyter_text>Train Process<jupyter_code># loss
criterion = nn.BCELoss() 

# optimizer
lr = 0.0002
b1 = 0.5
b2 = 0.999
G_optimizer = torch.optim.Adam(G.parameters(), lr=lr, betas=(b1, b2))
D_optimizer = torch.optim.Adam(D.parameters(), lr=lr, betas=(b1, b2))<jupyter_output><empty_output><jupyter_text>Discriminator Update ![title](https://miro.medium.com/max/1720/1*4xAHMaUGXeOQnNJhzjq-4Q.jpeg)  Generator Update  Before : ![title](https://miro.medium.com/max/1224/1*n235XEigXKL3ktL08d-CZA.jpeg)  Because Generator diminished gradient: In practice, equation 1 may not provide sufficient gradient for G to learn well.  Early in learning, when G is poor,D can reject samples with high confidence because they are clearly different fromthe training data.  In this case, log(1−D(G(z))) saturates.  Rather than training G to minimize log(1−D(G(z))) we can train G to maximize logD(G(z)). This objective function results in thesame fixed point of the dynamics of G and D but provides much stronger gradients early in learning. (GAN Paper)![title](https://miro.medium.com/max/1517/1*6So6q3dWurG8qrmwk1y3jw.jpeg)  ![BCEloss](https://github.com/DSC-UI-SRIN/Introduction-to-GAN/raw/master/1%20-%20Fundamental%20of%20GANs/images/BCELoss.png)<jupyter_code>Tensor = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor
epochs = 200
for epoch in range(epochs):
    for i, (imgs, _) in enumerate(train_loader):

        # Adversarial ground truths
        valid = Variable(Tensor(imgs.size(0), 1).fill_(1.0), requires_grad=False)
        fake = Variable(Tensor(imgs.size(0), 1).fill_(0.0), requires_grad=False)

        # Configure input
        real_imgs = Variable(imgs.type(Tensor))

        # -----------------
        #  Train Generator
        # -----------------

        G_optimizer.zero_grad()

        # Sample noise as generator input
        z = Variable(Tensor(np.random.normal(0, 1, (imgs.shape[0], z_dim))))

        # Generate a batch of images
        gen_imgs = G(z)

        # Loss measures generator's ability to fool the discriminator
#         g_loss = criterion(D(gen_imgs), fake) # Normal MinMax
        g_loss = criterion(D(gen_imgs), valid) # Non Saturated

        g_loss.backward()
        G_optimizer.step()

        # ---------------------
        #  Train Discriminator
        # ---------------------

        D_optimizer.zero_grad()

        # Measure discriminator's ability to classify real from generated samples
        real_loss = criterion(D(real_imgs), valid)
        fake_loss = criterion(D(gen_imgs.detach()), fake)
        d_loss = (real_loss + fake_loss) / 2

        d_loss.backward()
        D_optimizer.step()
        if i % 300 == 0:
            print(
              "[Epoch %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]"
              % (epoch, epochs, i, len(train_loader), d_loss.item(), g_loss.item()))

    if epoch % 5 == 0:
        save_image(gen_imgs.view(gen_imgs.size(0), 1, 28, 28), ROOT + "sample/%d.png" % epoch, nrow=5, normalize=True)

torch.save(G, ROOT + 'G.pt')
torch.save(D, ROOT + 'D.pt')<jupyter_output>[Epoch 0/200] [Batch 0/600] [D loss: 0.672629] [G loss: 0.718289]
[Epoch 0/200] [Batch 300/600] [D loss: 0.525954] [G loss: 1.035763]
[Epoch 1/200] [Batch 0/600] [D loss: 0.487692] [G loss: 0.853348]
[Epoch 1/200] [Batch 300/600] [D loss: 0.362092] [G loss: 0.910943]
[Epoch 2/200] [Batch 0/600] [D loss: 0.317175] [G loss: 1.666999]
[Epoch 2/200] [Batch 300/600] [D loss: 0.340499] [G loss: 1.058032]
[Epoch 3/200] [Batch 0/600] [D loss: 0.320626] [G loss: 1.648210]
[Epoch 3/200] [Batch 300/600] [D loss: 0.273704] [G loss: 1.251731]
[Epoch 4/200] [Batch 0/600] [D loss: 0.314576] [G loss: 1.135395]
[Epoch 4/200] [Batch 300/600] [D loss: 0.557768] [G loss: 0.554418]
[Epoch 5/200] [Batch 0/600] [D loss: 0.212887] [G loss: 1.849806]
[Epoch 5/200] [Batch 300/600] [D loss: 0.345930] [G loss: 1.052477]
[Epoch 6/200] [Batch 0/600] [D loss: 0.512333] [G loss: 3.889980]
[Epoch 6/200] [Batch 300/600] [D loss: 0.225671] [G loss: 1.580570]
[Epoch 7/200] [Batch 0/600] [D loss: 0.351296] [G loss: 2.8760[...]
{'fraction_non_alphanumeric': 0.15269777830022335, 'fraction_numerical': 0.07299870694721994, 'mean_word_length': 3.5766541151156535, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '17651', 'license': 'MIT', 'n_tokens_mistral': 3597, 'n_tokens_neox': 3111, 'n_words': 669, 'path': '1 - Fundamental of GANs/MinMax-NS.ipynb', 'repo_name': 'drainvers/Introduction-to-GAN', 'size': '8530'}