<jupyter_start><jupyter_text>Setup<jupyter_code>%matplotlib qt
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow_gan as tfgan
import numpy as np
import os, sys
from tqdm.notebook import tqdm
from pathlib import Path

sys.path.append( os.path.abspath('..') )
import utils
Path('Flowers').mkdir(exist_ok=True)
os.chdir('Flowers')<jupyter_output><empty_output><jupyter_text>Loading the preprocessed Flowers dataset, make sure to execute the code found at `Other/Preprocessing.ipynb` in order to create this data.<jupyter_code>dataset = tf.data.Dataset.from_tensor_slices(np.load(os.path.join('..', '..', 'flowers.npy')))
dataset = dataset.map(lambda img: (tf.cast(img, tf.float32) - 127.5) / 127.5)
NUM_IMAGES = int(dataset.cardinality())<jupyter_output><empty_output><jupyter_text>1 Models  1.1 Architecure In these experiments the use of Batch Normalization has shown to produce artifacts<jupyter_code>def generator_model_transp_conv(latent_dims):
    return tf.keras.Sequential([
        tf.keras.layers.Dense(6*6*512, input_shape=(latent_dims,)),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Reshape((6, 6, 512)),
        # Output Shape: 6x6x512
        
        tf.keras.layers.Conv2DTranspose(256, kernel_size=2, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 12x12x256
        
        tf.keras.layers.Conv2DTranspose(128, kernel_size=2, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 24x24x128
        
        tf.keras.layers.Conv2DTranspose(64, kernel_size=2, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 48x48x64
        
        tf.keras.layers.Conv2D(3, kernel_size=1, strides=1, padding='same', activation='tanh')
        # Output Shape: 48x48x3
    ])
def generator_model_upsample(latent_dims, interpolation):
    return tf.keras.Sequential([
        tf.keras.layers.Dense(6*6*512, input_shape=(latent_dims,)),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Reshape((6, 6, 512)),
        # Output Shape: 6x6x512
        
        tf.keras.layers.UpSampling2D(size=2, interpolation=interpolation),
        tf.keras.layers.Conv2D(256, kernel_size=3, strides=1, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 12x12x256
        
        tf.keras.layers.UpSampling2D(size=2, interpolation=interpolation),
        tf.keras.layers.Conv2D(128, kernel_size=3, strides=1, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 24x24x128
        
        tf.keras.layers.UpSampling2D(size=2, interpolation=interpolation),
        tf.keras.layers.Conv2D(64, kernel_size=3, strides=1, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        # tf.keras.layers.BatchNormalization(),
        # Output Shape: 48x48x64
        
        tf.keras.layers.Conv2D(3, kernel_size=1, strides=1, padding='same', activation='tanh')
        # Output Shape: 48x48x3
    ])
def discriminator_model():
    return tf.keras.Sequential([
        tf.keras.layers.Conv2D(64, kernel_size=1, strides=2, padding='same', input_shape=(48,48,3)),
        tf.keras.layers.LeakyReLU(0.2),
        tf.keras.layers.Dropout(0.3),
        # Output Shape: 48x48x64
        
        tf.keras.layers.Conv2D(128, kernel_size=3, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        tf.keras.layers.Dropout(0.3),
        # Output Shape: 24x24x128
        
        tf.keras.layers.Conv2D(256, kernel_size=3, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        tf.keras.layers.Dropout(0.3),
        # Output Shape: 12x12x256
        
        tf.keras.layers.Conv2D(512, kernel_size=3, strides=2, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        tf.keras.layers.Dropout(0.3),
        # Output Shape: 6x6x512

        tf.keras.layers.Conv2D(512, kernel_size=6, strides=1, padding='same'),
        tf.keras.layers.LeakyReLU(0.2),
        tf.keras.layers.Dropout(0.3),
        # Output Shape: 1x1x512

        tf.keras.layers.Dense(1)
    ])<jupyter_output><empty_output><jupyter_text>1.2 Losses The binary cross entropy (BCE) between $y$ and $\hat{y}$ is calculated as:$$    \mathrm{BCE}(y, \hat{y}) = - y \log\left(\hat{y}\right) - (1-y) \log\left(1 - \hat{y}\right)$$<jupyter_code>cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)<jupyter_output><empty_output><jupyter_text>The generator tries to maximize the chance of the discriminator being wrong. This is equivalent of trying to minimize the following loss function:$$    J^{(G)} = -\log\bigl(D\bigl(G(z)\bigr)\bigr)$$<jupyter_code>def generator_loss(fake_output):
    return cross_entropy(tf.ones_like(fake_output), fake_output)<jupyter_output><empty_output><jupyter_text>The discriminator tries to correctly classify real data as real and fake data as fake. This is equivalent to minimizing the following loss function:$$    J^{(D)} = -\log\bigr(D(x)\bigl) - \log\bigl(1 - D\bigl(G(z)\bigr)\bigr)$$Here we scale down the loss by a factor of $\;0.5$ and apply a one sided label smoothing of $\:0.9$<jupyter_code>def discriminator_loss(real_output, fake_output):
    real_loss = cross_entropy(0.9*tf.ones_like(real_output), real_output)
    fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)
    return 0.5 * (real_loss + fake_loss)<jupyter_output><empty_output><jupyter_text>2 Training  2.1 Main functions<jupyter_code>def discriminator_train_step(generator, discriminator, images, latent_dims):
    noise = tf.random.normal([images.shape[0], latent_dims])
    with tf.GradientTape() as disc_tape:
        generated_imgs = generator(noise, training=True)
        real_output = discriminator(images, training=True)
        fake_output = discriminator(generated_imgs, training=True)
        loss_D = discriminator_loss(real_output, fake_output)
    
    grads_D = disc_tape.gradient(loss_D, discriminator.trainable_variables)
    discriminator.optimizer.apply_gradients(zip(grads_D, discriminator.trainable_variables))
def generator_train_step(generator, discriminator, batch_size, latent_dims):
    noise = tf.random.normal([batch_size, latent_dims])
    with tf.GradientTape() as gen_tape:
        generated_imgs = generator(noise, training=True)
        fake_output = discriminator(generated_imgs, training=True)
        loss_G = generator_loss(fake_output)
    
    grads_G = gen_tape.gradient(loss_G, generator.trainable_variables)
    generator.optimizer.apply_gradients(zip(grads_G, generator.trainable_variables))
def train(generator, discriminator, dataset, epochs, batch_size, callbacks=None):
    latent_dims = generator.input_shape[1]
    num_batches = int(1 + (NUM_IMAGES - 1) // batch_size)
    
    generator_step = tf.function(generator_train_step)
    discriminator_step = tf.function(discriminator_train_step)
    for epoch in tqdm(range(epochs)):
        for c in callbacks:
            c.on_epoch_begin(epoch=epoch + 1, generator=generator, discriminator=discriminator)
        
        for batch in tqdm(dataset, leave=False, total=num_batches):
            discriminator_step(generator, discriminator, batch, latent_dims)
            generator_step(generator, discriminator, batch_size, latent_dims)
        
        for c in callbacks:
            c.on_epoch_end(epoch=epoch + 1, generator=generator, discriminator=discriminator)<jupyter_output><empty_output><jupyter_text>2.2 Hyperparameter Testing These were the hyperparameters tested for the final document. Training all of them simultaneously may take a long time, consider commenting out some options to run the tests individually.<jupyter_code>BATCH_SIZE = 24
LATENT_DIMS = 128
hparams_list = [
    {'upsample': 'TrpConv',  'epochs':  20},
    {'upsample': 'bilinear', 'epochs':  20},
    {'upsample': 'nearest',  'epochs': 100}
]
for hparams in hparams_list:
    dirname = '{}'.format(hparams['upsample'].upper())
    Path(dirname).mkdir(exist_ok=True)
    
    ## Models
    if hparams['upsample'] is 'TrpConv':
        generator = generator_model_transp_conv(LATENT_DIMS)
    else:
        generator = generator_model_upsample(LATENT_DIMS, hparams['upsample'])
    generator.optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.0)

    discriminator = discriminator_model()
    discriminator.optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.0)
    
    ## Callbacks
    timer = utils.callback.TimerCallback()
    save_samples = utils.callback.SaveSamplesCallback(
        path_format=os.path.join(dirname, 'epoch-{}'),
        inputs=tf.random.normal((8*8, LATENT_DIMS)),
        n_cols=8,
        savefig_kwargs={'bbox_inches': 'tight', 'pad_inches': 0, 'dpi': 256},
        grid_params={'border':2, 'pad':2, 'pad_value':0.0},
        transform_samples=lambda samples: (1 + samples) * 0.5
    )
    
    ## Train and save results
    train(
        generator,
        discriminator,
        dataset=dataset.shuffle(NUM_IMAGES).batch(BATCH_SIZE),
        epochs=hparams['epochs'],
        batch_size=BATCH_SIZE,
        callbacks=[timer, save_samples]
    )
    
    generator.save    (os.path.join(dirname, 'generator.h5'    ), overwrite=True, save_format='h5')
    discriminator.save(os.path.join(dirname, 'discriminator.h5'), overwrite=True, save_format='h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12773647253201156, 'fraction_numerical': 0.031598513011152414, 'mean_word_length': 3.719785575048733, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '477453', 'license': 'MIT', 'n_tokens_mistral': 3509, 'n_tokens_neox': 3333, 'n_words': 571, 'path': 'DCGAN/Flowers.ipynb', 'repo_name': 'PatrickHoeckler/tcc_gan', 'size': '9684'}