<jupyter_start><jupyter_text>Learning `Deep Convolutional GAN`<jupyter_code>import os
import sys
import datetime as dt

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

%matplotlib inline
from tensorflow.examples.tutorials.mnist import input_data
data = input_data.read_data_sets('datasets/MNIST', one_hot=True)<jupyter_output>Extracting datasets/MNIST/train-images-idx3-ubyte.gz
Extracting datasets/MNIST/train-labels-idx1-ubyte.gz
Extracting datasets/MNIST/t10k-images-idx3-ubyte.gz
Extracting datasets/MNIST/t10k-labels-idx1-ubyte.gz<jupyter_text>Plot a random image<jupyter_code>sample_image = data.test.images[3]
sample_image = sample_image.reshape([28, 28])
plt.imshow(sample_image, cmap='Greys', interpolation='bicubic')<jupyter_output><empty_output><jupyter_text>Hyperparams<jupyter_code># Inputs
img_size = 28
img_channel = 1
img_size_flat = img_size * img_size * img_channel

# Network
conv_filter = 5
deconv_filter = 3
strides = [1, 2, 2, 1]
conv_1 = 16
conv_2 = 32
fc_1_size = 1024
fc_2_size = 1

# Training
learning_rate = 1e-3
batch_size = 32
global_step = tf.Variable(0, name='global_step', trainable=False)<jupyter_output><empty_output><jupyter_text>Building the Network  Helper functions<jupyter_code># Weight initializer
def weight(shape, layer_name):
    return tf.get_variable(layer_name, shape, initializer=tf.truncated_normal_initializer(stddev=0.2))

# Bias initializer
def bias(shape, layer_name):
    return tf.get_variable(layer_name, [shape], initializer=tf.constant_initializer(0))

# Convolutional block
def conv_block(layer, W, b):
    layer = tf.nn.conv2d(layer, W, strides=[1, 1, 1, 1], padding='SAME')
    layer = layer + b  # add bias
    layer = tf.nn.relu(layer)
    layer = tf.nn.max_pool(layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
    return layer

# Deconvolutional block
def deconv_block(layer, W, b, activation_fn=tf.nn.relu, batch_norm=None):
    layer = tf.nn.conv2d(layer, W, strides=[1, 2, 2, 1], padding='SAME')
    layer = layer + b
    if batch_norm:
        layer = tf.contrib.layers.batch_norm(layer, epsilon=1e-5, scope=batch_norm)
    layer = activation_fn(layer)
    return layer

# Fully connected layer
def fully_connected(layer, W, b, use_relu=True):
    layer = tf.matmul(layer, W) + b
    if use_relu:
        layer = tf.nn.relu(layer)
    return layer

# Flatten layer
def flatten(layer):
    shape = layer.get_shape()
    features = np.array(shape[1:4], dtype=np.int32).prod()
    flattened = tf.reshape(layer, [-1, features])
    return flattened, features<jupyter_output><empty_output><jupyter_text>Discriminator `(Deep Convolutional neural net)`<jupyter_code>def discriminator(image, reuse=None):
    with tf.variable_scope(tf.get_variable_scope(), reuse=reuse) as scope:
        # 1st convolutional block
        d_conv_W1 = weight(shape=[conv_filter, conv_filter, img_channel, conv_1], layer_name='d_conv_W1')
        d_conv_b1 = bias(shape=conv_1, layer_name='d_conv_b1')
        d_conv_1 = conv_block(image, d_conv_W1, d_conv_b1)
        # 2nd convolutional block
        d_conv_W2 = weight(shape=[conv_filter, conv_filter, conv_1, conv_2], layer_name='d_conv_W2')
        d_conv_b2 = bias(shape=conv_2, layer_name='d_conv_b2')
        d_conv_2 = conv_block(d_conv_1, d_conv_W2, d_conv_b2)
        # Flatten layer
        flattened, features = flatten(d_conv_2)
        # 1st Fully connected layer
        d_fc_W1 = weight(shape=[features, fc_1_size], layer_name='d_fc_W1')
        d_fc_b1 = bias(shape=fc_1_size, layer_name='d_fc_b1')
        fc_1 = fully_connected(flattened, d_fc_W1, d_fc_b1, use_relu=True)
        # 2nd Fully connected layer
        d_fc_W2 = weight(shape=[fc_1_size, fc_2_size], layer_name='d_fc_W2')
        d_fc_b2 = bias(shape=fc_2_size, layer_name='d_fc_b2')
        fc_2 = fully_connected(fc_1, d_fc_W2, d_fc_b2, use_relu=False)
        return fc_2<jupyter_output><empty_output><jupyter_text>Generator `(Deep Deconvolutional neural net)`<jupyter_code>def generator(z, batch_size, z_dim, scale=56):
    up_scale = scale * scale
    # 1st Deconvolutional block
    g_deconv_W1 = weight(shape=[z_dim, up_scale], layer_name='g_deconv_W1')
    g_deconv_b1 = bias(shape=up_scale, layer_name='g_deconv_b1')
    g_deconv_1 = tf.matmul(z, g_deconv_W1) + g_deconv_b1
    g_deconv_1 = tf.reshape(g_deconv_1, [-1, scale, scale, img_channel])
    g_deconv_1 = tf.contrib.layers.batch_norm(g_deconv_1, epsilon=1e-5, scope='bn_G1')
    g_deconv_1 = tf.nn.relu(g_deconv_1)
    # 2nd Deconvolutional block
    g_deconv_W2 = weight(shape=[deconv_filter, deconv_filter, img_channel, z_dim//2], layer_name='g_deconv_W2')
    g_deconv_b2 = bias(shape=z_dim//2, layer_name='g_deconv_b2')
    g_deconv_2 = deconv_block(g_deconv_1, g_deconv_W2, g_deconv_b2, batch_norm='bn_G2')
    g_deconv_2 = tf.image.resize_images(g_deconv_2, [scale, scale])
    # 3rd Deconvolutional block
    g_deconv_W3 = weight(shape=[deconv_filter, deconv_filter, z_dim//2, z_dim//4], layer_name='g_deconv_W3')
    g_deconv_b3 = bias(shape=z_dim//4, layer_name='g_deconv_b3')
    g_deconv_3 = deconv_block(g_deconv_2, g_deconv_W3, g_deconv_b3, batch_norm='bn_G3')
    g_deconv_3 = tf.image.resize_images(g_deconv_3, [scale, scale])
    # Final Deconvolutional block
    g_deconv_W4 = weight(shape=[1, 1, z_dim//4, 1], layer_name='g_deconv_W4')
    g_deconv_b4 = bias(shape=1, layer_name='g_deconv_b4')
    g_deconv_4 = deconv_block(g_deconv_3, g_deconv_W4, g_deconv_b4, activation_fn=tf.nn.sigmoid)
    
    return g_deconv_4
z_dim = 100
z_placeholder = tf.placeholder(tf.float32, [None, z_dim])
gen_img = generator(z_placeholder, 1, z_dim)
z_batch = np.random.normal(0, 1, [1, z_dim])
init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    test_img = sess.run(gen_img, feed_dict={z_placeholder: z_batch})
    print(test_img.shape)
    plt.title('Test image')
    plt.imshow(test_img.reshape([img_size, img_size]), interpolation='bicubic', cmap='Greys')
tf.reset_default_graph()

# Placeholder
z_placeholder = tf.placeholder(tf.float32, [None, z_dim], name='z_placeholder')
X_placeholder = tf.placeholder(tf.float32, [None, img_size_flat], name='X_placeholder')
X_image = tf.reshape(X_placeholder, [-1, img_size, img_size, img_channel])

# Genenrated img
Gz = generator(z_placeholder, batch_size, z_dim)
# Discriminator
# Prob. for real img
Dx = discriminator(X_image)
# Prob. for generated img
Dg = discriminator(Gz, reuse=True)<jupyter_output><empty_output><jupyter_text>Loss functions<jupyter_code># Discriminator loss
d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=Dx, labels=tf.ones_like(Dx)))
d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=Dg, labels=tf.zeros_like(Dg)))
# Generator loss
g_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=Dg, labels=tf.ones_like(Dg)))<jupyter_output><empty_output><jupyter_text>Retrieve Trainable Vars<jupyter_code>t_vars = tf.trainable_variables()

d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]

print([var.name for var in d_vars])
print([var.name for var in g_vars])<jupyter_output>['d_conv_W1:0', 'd_conv_b1:0', 'd_conv_W2:0', 'd_conv_b2:0', 'd_fc_W1:0', 'd_fc_b1:0', 'd_fc_W2:0', 'd_fc_b2:0']
['g_deconv_W1:0', 'g_deconv_b1:0', 'g_deconv_W2:0', 'g_deconv_b2:0', 'g_deconv_W3:0', 'g_deconv_b3:0', 'g_deconv_W4:0', 'g_deconv_b4:0']<jupyter_text>Optimization<jupyter_code># Discriminator optimization
d_opt_real = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(d_loss_real, var_list=d_vars)
d_opt_fake = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(d_loss_fake, var_list=d_vars)
# Generator optimization
g_opt = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(g_loss, global_step=global_step, var_list=g_vars)<jupyter_output><empty_output><jupyter_text>Tensorboard stuff<jupyter_code>saver = tf.train.Saver()

# Henceforth, reuse variables
tf.get_variable_scope().reuse_variables()

# Summaries
tf.summary.scalar('Generator_Loss', g_loss)
tf.summary.scalar('Discriminator_Loss_Real', d_loss_real)
tf.summary.scalar('Discriminator_Loss_Fake', d_loss_fake)
img_4_tensorboard = generator(z_placeholder, batch_size, z_dim)
tf.summary.image('Generated_Images', img_4_tensorboard, max_outputs=5)
# merge all summaries
merged = tf.summary.merge_all()

# Directories
tensorboard_dir = 'tensorboard/'
logdir = os.path.join(tensorboard_dir, '{:%d-%b-%Y  %H-%M-%S-%p}'.format(dt.datetime.now()))
save_path = 'models/'

writer = tf.summary.FileWriter(logdir, sess.graph)

# Create checkpoint dir
if not tf.gfile.Exists(save_path):
    tf.gfile.MakeDirs(save_path)
sess = tf.Session()

# restore any checkpoints if it exists
try:
    sys.stdout.write('INFO: Trying to restore the latest checkpoint\n')
    last_chkpt = tf.train.latest_checkpoint(checkpoint_dir=save_path)
    saver.restore(sess=sess, save_path=last_chkpt)
    sys.stdout.write('INFO: Restored checkpoint from: {}'.format(last_chkpt))
except Exception as e:
    sys.stderr.write('ERR: {} Initializing variables instead.'.format(e))
    sess.run(tf.global_variables_initializer())
train_start = dt.datetime.now()
# Pre-train discriminator
for i in range(100):
    fake_img_batch = np.random.normal(0, 1, [batch_size, z_dim])
    real_img_batch = data.test.next_batch(batch_size)[0]
    sess.run([d_opt_fake, d_opt_real], feed_dict={X_placeholder: real_img_batch, 
                                                  z_placeholder: fake_img_batch})

    sys.stdout.write('\rIterations: {:,}\tTraining time: {}'.format(i+1, dt.datetime.now() - train_start))

# Train generator & discriminator together (dual each other)
train_start = dt.datetime.now()
for i in range(10000):
    real_img_batch = data.test.next_batch(batch_size)[0]
    fake_img_batch = np.random.normal(0, 1, [batch_size, z_dim])
    # Train discriminator
    sess.run([d_opt_fake, d_opt_real], feed_dict={X_placeholder: real_img_batch, 
                                                  z_placeholder: fake_img_batch})
    # Train generator
    gen_img = np.random.normal(0, 1, size=[batch_size, z_dim])
    _, i_global = sess.run([g_opt, global_step], feed_dict={z_placeholder: gen_img})
    
    # Updating tensorboard
    if i % 10 == 0:
        # save model chkpt
        saver.save(sess=sess, save_path=save_path, global_step=global_step)
        # update tensorboard
        gen_img = np.random.normal(0, 1, size=[batch_size, z_dim])
        summary = sess.run(merged, feed_dict={X_placeholder: real_img_batch,
                                              z_placeholder: gen_img})
        writer.add_summary(summary=summary, global_step=i_global)
    # Logging
    if i % 1000 == 0:
        # Every 1000 iterations, show a generated image
        print("\tIterations: {:,} at {:%a %h %d, %Y at %H:%M:%S %p}".format(i, dt.datetime.now()))
        gen_img = np.random.normal(0, 1, size=[1, z_dim])
        generated_images = generator(z_placeholder, 1, z_dim)
        images = sess.run(generated_images, {z_placeholder: gen_img})
        plt.imshow(images[0].reshape([img_size, img_size]), interpolation='bicubic', cmap='Greys')
        plt.show()

        # Show discriminator's estimate
        im = images[0]
        result = discriminator(tf.reshape(X_placeholder, [-1, img_size, img_size, img_channel]))
        estimate = sess.run(result, {X_placeholder: np.reshape(im, [-1, img_size_flat])})
        print("Estimate: {}\n".format(estimate))
    sys.stdout.write('\rIterations: {:,}\tGlobal steps: {:,}\tTraining time: {:}'.format(i+1, i_global, dt.datetime.now() - train_start))<jupyter_output>Iterations: 100	Training time: 0:00:21.193085
{'fraction_non_alphanumeric': 0.12770858215321618, 'fraction_numerical': 0.023886708752772566, 'mean_word_length': 4.968940936863544, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '484280', 'license': 'MIT', 'n_tokens_mistral': 4644, 'n_tokens_neox': 4476, 'n_words': 776, 'path': 'learning-gan.ipynb', 'repo_name': 'victor-iyiola/deep-convolutional-GAN', 'size': '11722'}