<jupyter_start><jupyter_text>Generative Adversarial NetworksThis code is based on [_Generative Adversarial Networks (2014)_](https://arxiv.org/abs/1406.2661) paper by <NAME>, <NAME>, et all.![title](https://github.com/DSC-UI-SRIN/Introduction-to-GAN/raw/master/1%20-%20Fundamental%20of%20GANs/images/minmax.png)  Environment Setup<jupyter_code>%load_ext pycodestyle_magic
%flake8_on
import torch

# device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("using %s" % (device))<jupyter_output>using cuda<jupyter_text>Prepare Dataset<jupyter_code>from torchvision import datasets, transforms

# setup MNIST dataset
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.5], [0.5])])

mnist_dataset = datasets.MNIST(
    root="./datasets/mnist", train=True, transform=transform, download=True)

mnist_width = mnist_dataset.train_data.size(1)
mnist_height = mnist_dataset.train_data.size(2)
mnist_shape = [1, mnist_width, mnist_height]

# data loader (input pipeline)
data_loader = torch.utils.data.DataLoader(
    dataset=mnist_dataset, batch_size=100, shuffle=True)
import matplotlib.pyplot as plt

examples = enumerate(data_loader)
_, (example_data, _) = next(examples)

# plot example data
for i in range(6):
    plt.subplot(2, 3, i+1)
    plt.tight_layout()
    plt.imshow(example_data[i][0], cmap='gray', interpolation='none')
    plt.title("Example %d" % i)
    plt.xticks([])
    plt.yticks([])

plt.show()<jupyter_output><empty_output><jupyter_text>Build Models![title](https://miro.medium.com/max/1600/1*M_YipQF_oC6owsU1VVrfhg.jpeg)source: https://medium.com/@jonathan_hui/gan-whats-generative-adversarial-networks-and-its-application-f39ed278ef09<jupyter_code>import torch.nn as nn


def nn_block(in_feat, out_feat, normalize=False):
    layers = [nn.Linear(in_feat, out_feat)]
    if normalize:
        layers.append(nn.BatchNorm1d(out_feat, 0.8))
    layers.append(nn.LeakyReLU(0.2, inplace=True))
    return layers
# generator model
class Generator(nn.Module):
    def __init__(self, in_dim, out_dim):
        super(Generator, self).__init__()

        self.model = nn.Sequential(
            *nn_block(in_dim, 128),
            *nn_block(128, 256, normalize=True),
            *nn_block(256, 512, normalize=True),
            *nn_block(512, 1024, normalize=True),
            nn.Linear(1024, out_dim),
            nn.Tanh())

    def forward(self, noise):
        output = self.model(noise)
        image = output.view(output.size(0), *mnist_shape)
        return image
# discriminator model
class Discriminator(nn.Module):
    def __init__(self, in_dim):
        super(Discriminator, self).__init__()

        self.model = nn.Sequential(
            *nn_block(in_dim, 512),
            *nn_block(512, 256),
            nn.Linear(256, 1),
            nn.Sigmoid())

    def forward(self, image):
        img_flat = image.view(image.size(0), -1)
        validity = self.model(img_flat)
        return validity
noise_dim = 100
mnist_dim = mnist_width * mnist_height

# build model
generator = Generator(in_dim=noise_dim, out_dim=mnist_dim).to(device)
discriminator = Discriminator(in_dim=mnist_dim).to(device)

print(generator, discriminator)<jupyter_output>Generator(
  (model): Sequential(
    (0): Linear(in_features=100, out_features=128, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace=True)
    (2): Linear(in_features=128, out_features=256, bias=True)
    (3): BatchNorm1d(256, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (4): LeakyReLU(negative_slope=0.2, inplace=True)
    (5): Linear(in_features=256, out_features=512, bias=True)
    (6): BatchNorm1d(512, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (7): LeakyReLU(negative_slope=0.2, inplace=True)
    (8): Linear(in_features=512, out_features=1024, bias=True)
    (9): BatchNorm1d(1024, eps=0.8, momentum=0.1, affine=True, track_running_stats=True)
    (10): LeakyReLU(negative_slope=0.2, inplace=True)
    (11): Linear(in_features=1024, out_features=784, bias=True)
    (12): Tanh()
  )
) Discriminator(
  (model): Sequential(
    (0): Linear(in_features=784, out_features=512, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace=True)[...]<jupyter_text>Train Process Discriminator Update![title](https://miro.medium.com/max/1720/1*4xAHMaUGXeOQnNJhzjq-4Q.jpeg) Generator Update Before:![title](https://miro.medium.com/max/1224/1*n235XEigXKL3ktL08d-CZA.jpeg) Because Generator diminished gradient:In practice, equation 1 may not provide sufficient gradient for G to learn well.  Early in learning, when G is poor,D can reject samples with high confidence because they are clearly different fromthe training data.  In this case, log(1−D(G(z))) saturates.  Rather than training G to minimize log(1−D(G(z))) we can train G to maximize logD(G(z)). This objective function results in thesame fixed point of the dynamics of G and D but provides much stronger gradients early in learning. (GAN Paper)![title](https://miro.medium.com/max/1517/1*6So6q3dWurG8qrmwk1y3jw.jpeg)![BCEloss](https://github.com/DSC-UI-SRIN/Introduction-to-GAN/raw/master/1%20-%20Fundamental%20of%20GANs/images/BCELoss.png)<jupyter_code># loss criterion
criterion = nn.BCELoss().to(device)

# optimizers
g_optimizer = torch.optim.Adam(
    generator.parameters(), lr=0.0002, betas=(0.5, 0.999))

d_optimizer = torch.optim.Adam(
    discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))

if torch.cuda.is_available():
    FloatTensor = torch.cuda.FloatTensor
else:
    FloatTensor = torch.FloatTensor
from torch.autograd import Variable
from torchvision.utils import save_image
import numpy as np
import time
import os

start = time.time()

g_losses = []
d_losses = []

# train process
epochs = 200
for epoch in range(epochs):

    g_loss_sum = 0
    d_loss_sum = 0

    for i, (imgs, _) in enumerate(data_loader):

        batch_size = imgs.size(0)

        # adversarial ground truths
        valid = Variable(FloatTensor(batch_size, 1).fill_(1.0),
                         requires_grad=False)

        fake = Variable(FloatTensor(batch_size, 1).fill_(0.0),
                        requires_grad=False)

        # configure input
        real_imgs = Variable(imgs.type(FloatTensor))

        # -----------------
        #  Train Generator
        # -----------------

        g_optimizer.zero_grad()

        # sample noise as generator input
        noise = Variable(FloatTensor(
            np.random.normal(0, 1, (batch_size, noise_dim))))

        # generate a batch of images
        gen_imgs = generator(noise)

        # loss measures generator's ability to fool the discriminator
        # g_loss = criterion(discriminator(gen_imgs), fake)  # Normal MinMax
        g_loss = criterion(discriminator(gen_imgs), valid)  # Non Saturated
        g_loss_sum += g_loss.item()

        g_loss.backward()
        g_optimizer.step()

        # ---------------------
        #  Train Discriminator
        # ---------------------

        d_optimizer.zero_grad()

        # loss measures discriminator's ability to classify real
        # from generated samples
        real_loss = criterion(discriminator(real_imgs), valid)
        fake_loss = criterion(discriminator(gen_imgs.detach()), fake)
        d_loss = (real_loss + fake_loss) / 2
        d_loss_sum += d_loss.item()

        d_loss.backward()
        d_optimizer.step()

        elapsed = time.time() - start
        print("\r[Elapsed %02d:%02d]" % (elapsed // 60, elapsed % 60), end='')

        print("\r[Epoch %d/%d] [Batch %d/%d]"
              % (epoch+1, epochs, i+1, len(data_loader)), end='')

        print(" [G loss: %f] [D loss: %f]"
              % (g_loss.item(), d_loss.item()), end='')

    g_losses.append(g_loss_sum / len(data_loader))
    d_losses.append(d_loss_sum / len(data_loader))

    if (epoch + 1) % 10 == 0:
        # sample noise as generator input
        noise = Variable(FloatTensor(
            np.random.normal(0, 1, (5 * 5, noise_dim))))

        # generate a batch of sample images
        gen_imgs = generator(noise)

        # save sample images
        os.makedirs("./samples/gan", exist_ok=True)
        save_image(gen_imgs, "./samples/gan/sample_%03d.png" % (epoch + 1),
                   nrow=5, normalize=True)

# save result models
os.makedirs("./models/gan", exist_ok=True)
torch.save(generator, "./models/gan/generator.pt")
torch.save(discriminator, "./models/gan/discriminator.pt")<jupyter_output>[Epoch 200/200] [Batch 600/600] [G loss: 3.215877] [D loss: 0.207704]<jupyter_text>Train Result<jupyter_code># plot losses
plt.plot(g_losses, label="generator loss")
plt.plot(d_losses, label="discriminator loss")
plt.legend()
plt.show()
# sample noise as generator input
noise = Variable(FloatTensor(np.random.normal(0, 1, (6, noise_dim))))

# generate a batch of sample images
gen_imgs = generator(noise)

# plot sample images
imgs = gen_imgs.cpu().detach().numpy()
for i in range(6):
    plt.subplot(2, 3, i+1)
    plt.tight_layout()
    plt.imshow(imgs[i][0], cmap='gray', interpolation='none')
    plt.title("Sample %d" % i)
    plt.xticks([])
    plt.yticks([])

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1417065737919025, 'fraction_numerical': 0.03548106225511537, 'mean_word_length': 3.5557759048091224, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '496519', 'license': 'MIT', 'n_tokens_mistral': 3465, 'n_tokens_neox': 3158, 'n_words': 635, 'path': '1 - Fundamental of GANs/GAN.ipynb', 'repo_name': 'threeal/Introduction-to-GAN', 'size': '9211'}