<jupyter_start><jupyter_text>Variational AutoEncoder 연습   필요한 모듈 import* tensorflow (신경망 구성, 변수 설정, 역전파, gradient descent etc)* numpy (텐서 연산)* matplotlib (결과 그래프 출력)<jupyter_code>import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>MNIST 데이터세트 로딩<jupyter_code># Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)<jupyter_output><empty_output><jupyter_text>하이퍼 파라메터 설정* hyper parameter는 사용자가 정의하는 변수들 * learnable parameter는 학습을 통해 업데이트되는 변수들<jupyter_code># Training Parameters
learning_rate = 0.001
num_steps = 50000
batch_size = 64<jupyter_output><empty_output><jupyter_text>러너블 파라메터 설정 (learnable parameter = network parameter)                           * 784 (input) $\rightarrow$ 512 (hidden) $\rightarrow$ 2(mean) + 2(std) $\rightarrow$ 2 (latent)  $\rightarrow$ 512 (hidden) $\rightarrow$ 784 (output)<jupyter_code># Network Parameters
image_dim = 784 # MNIST images are 28x28 pixels
hidden_dim = 512
latent_dim = 2

# A custom initialization (see Xavier Glorot init)
def glorot_init(shape):
    return tf.random_normal(shape=shape, stddev=1. / tf.sqrt(shape[0] / 2.))

# tf Graph input (28x28 = 784 image size)
input_image = tf.placeholder(tf.float32, shape=[None, image_dim])

# Variables
weights = {
    'encoder_h1': tf.Variable(glorot_init([image_dim, hidden_dim])),
    'z_mean': tf.Variable(glorot_init([hidden_dim, latent_dim])),
    'z_std': tf.Variable(glorot_init([hidden_dim, latent_dim])),
    'decoder_h1': tf.Variable(glorot_init([latent_dim, hidden_dim])),
    'decoder_out': tf.Variable(glorot_init([hidden_dim, image_dim]))
}

biases = {
    'encoder_b1': tf.Variable(glorot_init([hidden_dim])),
    'z_mean': tf.Variable(glorot_init([latent_dim])),
    'z_std': tf.Variable(glorot_init([latent_dim])),
    'decoder_b1': tf.Variable(glorot_init([hidden_dim])),
    'decoder_out': tf.Variable(glorot_init([image_dim]))
}<jupyter_output><empty_output><jupyter_text>Encoder 부분 구축<jupyter_code># Building the encoder
def encoder(x):

    # encoder hidden layer 512
    en_hidden = tf.matmul(x, weights['encoder_h1']) + biases['encoder_b1']
    en_hidden = tf.nn.tanh(en_hidden)

    # latent layer 2
    z_mean = tf.matmul(en_hidden, weights['z_mean']) + biases['z_mean']
    z_std = tf.matmul(en_hidden, weights['z_std']) + biases['z_std']

    # Sampler: Normal (gaussian) random distribution
    eps = tf.random_normal(tf.shape(z_std), dtype=tf.float32, mean=0., stddev=1.0,
                       name='epsilon')
    z = z_mean + tf.exp(z_std / 2) * eps

    return z_mean, z_std, z<jupyter_output><empty_output><jupyter_text>Decoder 부분 구축<jupyter_code># Building the decoder (with scope to re-use these layers later)
def decoder(z):

    # decoder hidden layer 512
    de_hidden = tf.matmul(z, weights['decoder_h1']) + biases['decoder_b1']
    de_hidden = tf.nn.tanh(de_hidden)
    reconstruct = tf.matmul(de_hidden, weights['decoder_out']) + biases['decoder_out']
    reconstruct = tf.nn.sigmoid(reconstruct)
    
    return reconstruct<jupyter_output><empty_output><jupyter_text>Reconstruction Loss  KL Divergence (Regularization) Loss<jupyter_code># Define VAE Loss
    
# latent computation
mean, std, latent = encoder(input_image)

# Reconstruct image from latent
reconstruct = decoder(latent)
    
# Reconstruction loss (binary cross entropy)
reconstruct_loss = input_image * tf.log(1e-10 + reconstruct) \
                   + (1 - input_image) * tf.log(1e-10 + 1 - reconstruct)
reconstruct_loss = -tf.reduce_sum(reconstruct_loss, 1)
    
# KL Divergence (regularization)loss
kl_div_loss = 1 + std - tf.square(mean) - tf.exp(std)
kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1)

total_loss = tf.reduce_mean(reconstruct_loss + kl_div_loss)

optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(total_loss)

# Start training
sess = tf.Session()

# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()

# Run the initializer
sess.run(init)

for i in range(1, num_steps+1):
    # Prepare Data
    # Get the next batch of MNIST data (only images are needed, not labels)
    batch_x, _ = mnist.train.next_batch(batch_size)

    # Train
    feed_dict = {input_image: batch_x}
    _, l = sess.run([train_op, total_loss], feed_dict=feed_dict)
    
    if i % 1000 == 0 or i == 1:
        print('Step %i, Loss: %f' % (i, l))

# Testing
# Generator takes noise as input
noise_input = tf.placeholder(tf.float32, shape=[None, latent_dim])

# Rebuild the decoder to create image from noise
decoder = tf.matmul(noise_input, weights['decoder_h1']) + biases['decoder_b1']
decoder = tf.nn.tanh(decoder)
decoder = tf.matmul(decoder, weights['decoder_out']) + biases['decoder_out']
decoder = tf.nn.sigmoid(decoder)


# Building a manifold of generated digits
n = 5
x_axis = np.linspace(-3, 3, n)
y_axis = np.linspace(-3, 3, n)

canvas = np.empty((28 * n, 28 * n))
for i, yi in enumerate(x_axis):
    for j, xi in enumerate(y_axis):
        z_mu = np.array([[xi, yi]] * batch_size)
        recon = sess.run(decoder, feed_dict={noise_input: z_mu})
        canvas[(n - i - 1) * 28:(n - i) * 28, j * 28:(j + 1) * 28] = recon[0].reshape(28, 28)

plt.figure(figsize=(8, 10))
Xi, Yi = np.meshgrid(x_axis, y_axis)
plt.imshow(canvas, origin="upper", cmap="gray")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13190240322876537, 'fraction_numerical': 0.022381214456063107, 'mean_word_length': 4.696969696969697, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '613048', 'license': 'MIT', 'n_tokens_mistral': 2198, 'n_tokens_neox': 2120, 'n_words': 444, 'path': 'study2/1_generative_models/VariationalAutoEncoder.ipynb', 'repo_name': 'aidentify/lecture', 'size': '5451'}