<jupyter_start><jupyter_text>LAB9 Batch Normalization을 적용해보자목적: Batch Normalization을 이용하여 훈련속도를 개선해 보자<jupyter_code># preprocessor parts

from __future__ import absolute_import, division, print_function, unicode_literals

# Install TensorFlow
try:
  # %tensorflow_version only exists in Colab.
  %tensorflow_version 2.x
except Exception:
  pass

import tensorflow as tf
from tensorflow.keras.callbacks import TensorBoard

import numpy as np
import pandas as pd

from matplotlib import cm
from matplotlib import gridspec
import matplotlib.pyplot as plt

from datetime import datetime

from google.colab import drive
# This will prompt for authorization.
drive.mount('/content/drive')
  # for Tensorboard use
  LOG_DIR = 'content/drive/data/tb_logs'
	
  !wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
  !unzip ngrok-stable-linux-amd64.zip
	
  import os
  if not os.path.exists(LOG_DIR):
    os.makedirs(LOG_DIR)
	  
  get_ipython().system_raw(
      'tensorboard --logdir {} --host 0.0.0.0 --port 6006 &'
      .format(LOG_DIR))
	
  get_ipython().system_raw('./ngrok http 6006 &')
	
  !curl -s http://localhost:4040/api/tunnels | python3 -c \
      "import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])"
xsize  = 2
ysize  = 2

data = pd.read_csv('/content/drive/My Drive/seminar_materials/TensorFlow Lecture/KIT2020/data/lab45/twospirals_N5000.csv')
data.columns=['xdata1','xdata2','tdata']
permutation_index = np.random.permutation(data.index)
permutated_data = data.reindex(permutation_index)
permutated_data.columns=['xdata1','xdata2','tdata']

x_data = np.zeros([permutated_data.xdata1.size,xsize])
x_data[:,0] = permutated_data.xdata1.values
x_data[:,1] = permutated_data.xdata2.values

t_data = np.zeros([permutated_data.tdata.size,ysize])
t_data[:,0] = permutated_data.tdata.values
t_data[:,1] = np.invert(permutated_data.tdata.values) + 2


total_size = permutated_data.xdata1.size
training_size = int(np.floor(permutated_data.xdata1.size * 0.8))
validation_size = total_size - training_size



# data dividing
x_train = x_data[0:training_size,:]
y_train = t_data[0:training_size,:]

x_test = x_data[training_size:-1,:]
y_test = t_data[training_size:-1,:]

# #data plot
hfig1= plt.figure(1,figsize=[5,5])
plt.scatter(data.xdata1.values[0:int(data.xdata1.size/2)],\
            data.xdata2.values[0:int(data.xdata1.size/2)], \
            color='b',label='class1')
plt.scatter(data.xdata1.values[int(data.xdata1.size/2)+2:-1],\
            data.xdata2.values[int(data.xdata1.size/2)+2:-1], \
            color='r',label='class0')
plt.title('Cluster in Cluster data Example')
plt.legend()

plt.show()
# model building and training setting part

# Network Parameters
n_hidden_1 = 16 # 1st layer number of neurons
n_hidden_2 = 16 # 2nd layer number of neurons
n_hidden_3 = 16 # 3rd layer number of neurons
n_hidden_4 = 24 # 4rd layer number of neurons
n_hidden_5 = 24 # 5rd layer number of neurons
n_hidden_6 = 16 # 1st layer number of neurons
n_hidden_7 = 12 # 2nd layer number of neurons
n_hidden_8 = 7 # 3rd layer number of neurons
n_hidden_9 = 4 # 4rd layer number of neurons
n_hidden_10 = 4 # 5rd layer number of neurons
num_input = xsize   # two-dimensional input X = [1x2]
num_classes = ysize # 2 class

model = tf.keras.models.Sequential([
  tf.keras.layers.Dense(n_hidden_1, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_2, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_3, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_4, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_5, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_6, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_7, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_8, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_9, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(n_hidden_10, activation='relu'),
  tf.keras.layers.BatchNormalization(),
  tf.keras.layers.Dense(num_classes, activation='softmax')
])

opt_fn = tf.keras.optimizers.Adam(learning_rate=1e-5,
                                beta_1=0.9,
                                beta_2=0.999)


model.compile(optimizer=opt_fn,
              loss='categorical_crossentropy',
              metrics=['accuracy'])

tensorboard_callback = TensorBoard(log_dir=LOG_DIR, 
                           histogram_freq=1,
                           write_graph=True,
                           write_images=True)
# model training and evaluation part
training_epochs = 5
batch_size = 128

model.fit(x_train, y_train, 
          epochs=training_epochs,
          validation_data=(x_test, y_test),
          batch_size=batch_size,
          callbacks=[tensorboard_callback])
model.evaluate(x_test,  y_test, verbose=2)<jupyter_output>Train on 3999 samples, validate on 999 samples
Epoch 1/5
3999/3999 [==============================] - 3s 772us/sample - loss: 0.7513 - accuracy: 0.5469 - val_loss: 0.6929 - val_accuracy: 0.5075
Epoch 2/5
3999/3999 [==============================] - 0s 59us/sample - loss: 0.7391 - accuracy: 0.5564 - val_loss: 0.6871 - val_accuracy: 0.5115
Epoch 3/5
3999/3999 [==============================] - 0s 58us/sample - loss: 0.7284 - accuracy: 0.5594 - val_loss: 0.6808 - val_accuracy: 0.5255
Epoch 4/5
3999/3999 [==============================] - 0s 79us/sample - loss: 0.7186 - accuracy: 0.5754 - val_loss: 0.6828 - val_accuracy: 0.5365
Epoch 5/5
3999/3999 [==============================] - 0s 58us/sample - loss: 0.7099 - accuracy: 0.5789 - val_loss: 0.6864 - val_accuracy: 0.5696
999/999 - 0s - loss: 0.6864 - accuracy: 0.5696
{'fraction_non_alphanumeric': 0.15020297699594046, 'fraction_numerical': 0.05751014884979702, 'mean_word_length': 4.071183533447685, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '158582', 'license': 'Apache-2.0', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2168, 'n_words': 402, 'path': 'lab9_batchnorm_tf2.ipynb', 'repo_name': 'jwkanggist/EverybodyTensorflow2.0', 'size': '5912'}