<jupyter_start><jupyter_text>Import Tensorflow and other libraries<jupyter_code>from __future__ import absolute_import, division, print_function
# —Preview TF 2.0 Alpha build with GPU support (unstable, Ubuntu and Windows)
!pip install -q tensorflow-gpu==2.0.0-alpha0
import tensorflow as tf
tf.__version__
# To generate GIFs
!pip install -q imageio
import glob # The glob module finds all the pathnames matching a specified pattern according to the rules used by the Unix shell, although results are returned in arbitrary order. 
import imageio # Imageio is a Python library that provides an easy interface to read and write a wide range of image data, including animated images, volumetric data, and scientific formats. 
%matplotlib inline 
import matplotlib.pyplot as plt 
import numpy as np
import os # This module provides a portable way of using operating system dependent functionality. 
import PIL # DescriptionPython Imaging Library is a free library for the Python programming language that adds support for opening, manipulating, and saving many different image file formats.
import tensorflow.keras.layers as layers # Keras layers API
import time
from IPython import display # For displaying image
# Check if we're using a GPU device
# See what GPU we're using 
from tensorflow.python.client import device_lib

device_lib.list_local_devices()[-1]<jupyter_output><empty_output><jupyter_text>Load and prepare the datasetWe will use the MNIST dataset to train the gen and disc.<jupyter_code># Files will be located in "/content/drive/My Drive"
import os
%cd storage/
# image location
image_directory = os.getcwd() + "/img_align_celeba/"
ls
what = os.listdir(image_directory) # dir is your directory path
number_files = len(what)
print (number_files)
# Helper function to load image
def load_image( infilename ) :
    img = PIL.Image.open( infilename )
    img = img.resize((64,64))
    img.load()
    data = np.asarray( img, dtype="int32" )
    return data
# load the celebA data
test = load_image(image_directory + "000034.jpg")
test.shape
plt.imshow(test, interpolation='nearest')
plt.show()
files = os.listdir(image_directory)
train_images = []
for name in files:
    train_images.append(name)
# helper function to load images into an array
def load_images(file_names):
  data = []
  for name in file_names:
    data.append(load_image(image_directory + name))
  
  data = np.array(data)
  
  return data
# inspect the data real quick
test.shape # 218 x 178 x RGB
test[0][0]
#train_images = train_images.reshape(train_images.shape[0], 28, 28, 1). astype('float32') # puts each number in its own numpy array so instead of [1,2,3] gonna be [[1], [2], [3]]
#train_images = (train_images - 127.5) / 127.5 # Normalize the images to [-1. 1] so if it was 0 --> -1 and if it was 255 --> 1
BUFFER_SIZE = 12800 # number of images in training i think
BATCH_SIZE = 128 # This is just the standard number for batch size. Google for more info
# Shuffle and Batch the data
#train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
train_images = np.array(train_images)
np.random.shuffle(train_images)
train_images = np.split(train_images[:BUFFER_SIZE],BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Create the modelsBoth the generator and are definied using the Keras Sequential API<jupyter_code>!pip install -q pyyaml  # Required to save models in YAML format
def make_generator_model():
  # In Keras, you assemble layers to build models. A model is (usually) a graph of layers. The most common type of model is a stack of layers
  model = tf.keras.Sequential()
  
  """
  Add a densely-connected layer to the model
  the model will take as input arrays of shape (*, 100).
  And and output arrays of shape (*, 7*7*256)
  
  after the first layer, you don't need to specify the size of the input anymore
  Afterwards, we do automatic shape inference
  """
  model.add(layers.Dense(4*4*1024, use_bias = False, input_shape = (100,)))
  
  """
  You can think about batch normalization as doing preprocessing at every layer of the network.
  Accelerating Deep Network Training by Reducing Internal Covariate Shift
  """
  model.add(layers.BatchNormalization())
  
  """
  ReLU is linear (identity) for all positive values, and zero for all negative values. 
  Leaky ReLU has a small slope for negative values, instead of altogether zero. For example, leaky ReLU may have y = 0.01x when x < 0
  """
  model.add(layers.LeakyReLU())
  
  # reshape the output from something flattened to something with a shape of (7,7,256)
  model.add(layers.Reshape(( 4, 4, 1024)))
  #print(model.output_shape)
  #print("here 1")
  assert model.output_shape == (None, 4, 4, 1024) # Note: None is the batch size
  
  """
  The generator uses a transposed convolutional layer (Upsampling) layers to produce an image from seed (random noise).
  
  128 is the dimensionality of the output space
  (5,5) specifies the height and width of the 2D convolution window
  strides = (1,1) specifies the strides of the convolution along the height and width 
  
  """
  model.add(layers.Conv2DTranspose(512, (5, 5), strides = (2,2), padding = "same", use_bias = False))
  #print(model.output_shape)
  #print("here2")
  assert model.output_shape == (None, 8, 8, 512)
  model.add(layers.BatchNormalization())
  model.add(layers.LeakyReLU())
  
  # Another transposed convolutional layer (upsampling)
  #print("here3")
  model.add(layers.Conv2DTranspose(256, (5,5), strides = (2,2), padding = "same", use_bias = False))
  #print("here4")
  assert model.output_shape == (None, 16, 16, 256)
  model.add(layers.BatchNormalization())
  model.add(layers.LeakyReLU())
  
  # Another transposed convolutional layer (upsampling)
  #print("here4")
  model.add(layers.Conv2DTranspose(128, (5,5), strides = (2,2), padding = "same", use_bias = False))
  #print("here5")
  assert model.output_shape == (None, 32, 32, 128)
  model.add(layers.BatchNormalization())
  model.add(layers.LeakyReLU())
  
  # Final output layer also a convolutional layer (upsampling), tanh goes from -1 to 1
  #print("here6")
  model.add(layers.Conv2DTranspose(3, (5,5), strides = (2,2), padding = "same", use_bias = False, activation = "tanh"))
  #print("here7")

  
  print(model.output_shape)
  assert model.output_shape == (None, 64, 64, 3)
  
  
  return model<jupyter_output><empty_output><jupyter_text>Use the untrained generator to create an image<jupyter_code>generator = make_generator_model()

noise = tf.random.normal([1,100]) # shape is 1, 100
generated_image = generator(noise, training = False)

plt.imshow(generated_image[0, :, :, 0], cmap ="gist_rainbow")
type(generated_image[0, :, :, 0])<jupyter_output><empty_output><jupyter_text>The Discriminator<jupyter_code>def make_discriminator_model():
    model = tf.keras.Sequential()
    model.add(layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same', 
                                     input_shape=[64, 64, 3]))
    model.add(layers.LeakyReLU())
    model.add(layers.Dropout(0.3)) # Dropout consists in randomly setting a fraction rate of input units to 0 at each update during training time, which helps prevent overfitting.
      
    model.add(layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same'))
    model.add(layers.LeakyReLU())
    model.add(layers.Dropout(0.3))
    
    """
    Flattens the input. Does not affect the batch size.
    If inputs are shaped (batch,) without a channel dimension, then flattening adds an extra channel dimension and output shapes are (batch, 1).
    """
    model.add(layers.Flatten())
    model.add(layers.Dense(1))
     
    return model<jupyter_output><empty_output><jupyter_text>Use the untrained discriminator to classify the generated images as real or fake. The model will be trained to output positive values for real images, and negative values for fake images.<jupyter_code>discriminator = make_discriminator_model()
decision = discriminator(generated_image)
print(decision)<jupyter_output>tf.Tensor([[5.5305376e-05]], shape=(1, 1), dtype=float32)<jupyter_text>Define the Loss and Optimizers for both models<jupyter_code># This method returns a helper function to compute cross entropy loss
cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits = True)
"""
Discriminator Loss

This method quantifies how well the discriminator is able to distinguish real images from fakes. It compares the discriminator's predicitions on real images to an array of 1s
and the dicriminator's predicitons on fake (generated) images to an array of 0s.
"""
def discriminator_loss(real_output, fake_output):
  real_loss = cross_entropy(tf.ones_like(real_output), real_output)
  fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)
  total_loss = real_loss + fake_loss
  
  return total_loss
"""
Generator Loss

The generator's loss quantifies how well it was able to trick the discrimator. Intuitively, if the generator is performing well, the discriminator will classify the fake images as real (or 1).
Here, we will compare the discriminators decisions on the generated images to an array of 1s.
"""
def generator_loss(fake_output):
  return cross_entropy(tf.ones_like(fake_output), fake_output)
"""
The discriminator and the generator optimizers are different since we will train two networks separately.
The Adam optimization algorithm is an extension to stochastic gradient descent.
Stochastic gradient descent maintains a single learning rate (termed alpha) for all weight updates and the learning rate does not change during training.
A learning rate is maintained for each network weight (parameter) and separately adapted as learning unfolds.

"""
generator_optimizer = tf.keras.optimizers.Adam(1e-4)
discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)<jupyter_output><empty_output><jupyter_text>Save checkpointsDemonstrates how to save and restore models<jupyter_code>checkpoint_dir = "./training_checkpoints"
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
checkpoint = tf.train.Checkpoint(generator_optimizer = generator_optimizer,
                                 discriminator_optimizer = discriminator_optimizer,
                                 generator = generator,
                                 discriminator = discriminator)
type(generator)
##*****save generator
generator.save_weights('./training_checkpoints/md')
generator1  = make_generator_model()
generator1.load_weights('./training_checkpoints/md')
noise = tf.random.normal([1,100]) # shape is 1, 100
generated_image = generator1(noise, training = False)

plt.imshow(generated_image[0, :, :, 0], cmap ="gist_rainbow")<jupyter_output><empty_output><jupyter_text>Define the training loop<jupyter_code>EPOCHS = 50
noise_dim = 100
num_examples_to_generate = 8

# We will reuse this seed overtime (so it's easier) to visualize progress in the animated GIF
seed = tf.random.normal([num_examples_to_generate, noise_dim])
"""
The training loop begins with generator receiving a random seed as input. 
That seed is used to produce an image. The discriminator is then used to classify real images (drawn from the training set) and fakes images (produced by the generator). 
The loss is calculated for each of these models, and the gradients are used to update the generator and discriminator.
"""

# Notice the use of tf.function
# This annotation causes the function to be "compiled"
<EMAIL>
def train_step(images):
    noise = tf.random.normal([BATCH_SIZE, noise_dim])
    
    # take the image links and return a cropped image
    new_images = []
    #print("here1 ...")
    i = 0 
    for file_name in images:
        print(i)
        #print("Hey here")
        print(file_name)
        i+=1
        new_pic = load_image(image_directory + file_name)
        #print("ok")
        new_images.append(new_pic)
        #print("now what")
    
    #print("here 2")
    images = np.array(new_images)
    #print("reconfiguring complete")
    # Do some reshaping
    images = images.reshape(images.shape[0], 64, 64, 3).astype('float32') # puts each number in its own numpy array so instead of [1,2,3] gonna be [[1], [2], [3]]
    #print("here 3")
    images = (images - 127.5) / 127.5 # Normalize the images to [-1. 1] so if it was 0 --> -1 and if it was 255 --> 1
    print("Normalized the data")
    """
GradientTape() Records operations for automatic differentiation. Operations are recorded if 
they are executed within this context manager and at least one of their inputs is being "watched".
    """
    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
      generated_images = generator(noise, training=True)

      real_output = discriminator(images, training=True)
      fake_output = discriminator(generated_images, training=True)

      gen_loss = generator_loss(fake_output)
      disc_loss = discriminator_loss(real_output, fake_output)

    gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)
    gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)

    generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables)) # The zip() function returns an iterator of tuples based on the iterable object.
    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))
    
    # free memory!
    images = None
def train(dataset, epochs):  
  for epoch in range(epochs):
    start = time.time()
    
    for image_batch in dataset:
      #print(image_batch)
      train_step(image_batch)

    # Produce images for the GIF as we go
    display.clear_output(wait=True)
    generate_and_save_images(generator,
                             epoch + 1,
                             seed)
    
    # Save the model every 15 epochs
    if (epoch + 1) % 15 == 0:
      checkpoint.save(file_prefix = checkpoint_prefix)
    
    print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))
    
  # Generate after the final epoch
  display.clear_output(wait=True)
  generate_and_save_images(generator,
                           epochs,
                           seed)
image_directory<jupyter_output><empty_output><jupyter_text>Generate and Save the images<jupyter_code>def generate_and_save_images(model, epoch, test_input):
  # Notice `training` is set to False. 
  # This is so all layers run in inference mode (batchnorm).
  predictions = model(test_input, training=False)

  fig = plt.figure(figsize=(4,4))
  
  for i in range(predictions.shape[0]):
      plt.subplot(4, 4, i+1)
      plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5)
      plt.axis('off')
        
  plt.savefig('image_at_epoch_{:04d}.png'.format(epoch), cmap ="gist_rainbow")
  plt.show()<jupyter_output><empty_output><jupyter_text>Train the model  Call the train() method defined above to train the generator and discriminator simultaneously. Note, training GANs can be tricky. It's important that the generator and discriminator do not overpower each other (e.g., that they train at a similar rate).At the beginning of the training, the generated images look like random noise. As training progresses, the generated digits will look increasingly real. After about 50 epochs, they resemble MNIST digits. This may take about one minute / epoch with the default settings on Colab.<jupyter_code>%%time
train(train_images, EPOCHS)<jupyter_output><empty_output><jupyter_text>Restore the last checkpoint<jupyter_code>checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))<jupyter_output><empty_output><jupyter_text>Create a Gif<jupyter_code># Display a single image using the epoch number
def display_image(epoch_no):
  return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))
display_image(EPOCHS)<jupyter_output><empty_output><jupyter_text>Use imageio to create an animated gif using the images saved during training.<jupyter_code>with imageio.get_writer('dcgan.gif', mode='I') as writer:
  filenames = glob.glob('image*.png')
  filenames = sorted(filenames)
  last = -1
  for i,filename in enumerate(filenames):
    frame = 2*(i**0.5)
    if round(frame) > round(last):
      last = frame
    else:
      continue
    image = imageio.imread(filename)
    writer.append_data(image)
  image = imageio.imread(filename)
  writer.append_data(image)
    
# A hack to display the GIF inside this notebook
os.rename('dcgan.gif', 'dcgan.gif.png')
display.Image(filename="dcgan.gif.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09528147203497905, 'fraction_numerical': 0.019614987550859296, 'mean_word_length': 4.362422663627483, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '193144', 'license': 'MIT', 'n_tokens_mistral': 5269, 'n_tokens_neox': 4948, 'n_words': 1640, 'path': 'DCGAN_CelebA.ipynb', 'repo_name': 'skywall34/BabyGan', 'size': '16473'}