<jupyter_start><jupyter_text>MNIST convolutional neural networks* Make a networks like LeNet5 structure with MNIST data* input pipeline: `tf.data`* `Eager execution`* `Functional API`  Import modules<jupyter_code>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import os
import time

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from IPython.display import clear_output

import tensorflow as tf
from tensorflow.keras import layers
tf.enable_eager_execution()

os.environ["CUDA_VISIBLE_DEVICES"]="0"
print('TensorFlow version: {}'.format(tf.__version__))
# if you have a memeory trouble then uncomment it
# from tensorflow.compat.v1 import ConfigProto
# from tensorflow.compat.v1 import InteractiveSession

# config = ConfigProto()
# config.gpu_options.allow_growth = True
# session = InteractiveSession(config=config)<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code># Load training and eval data from tf.keras
(train_data, train_labels), (test_data, test_labels) = \
    tf.keras.datasets.mnist.load_data()

train_data = train_data / 255.
train_data = train_data.reshape([-1, 28, 28, 1])
train_data = train_data.astype(np.float32)
train_labels = train_labels.astype(np.int32)

test_data = test_data / 255.
test_data = test_data.reshape([-1, 28, 28, 1])
test_data = test_data.astype(np.float32)
test_labels = test_labels.astype(np.int32)<jupyter_output><empty_output><jupyter_text>Show the MNIST<jupyter_code>index = 219
print("label = {}".format(train_labels[index]))
plt.imshow(train_data[index][...,0])
plt.colorbar()
#plt.gca().grid(False)
plt.show()<jupyter_output><empty_output><jupyter_text>Set up dataset with `tf.data` input pipeline `tf.data.Dataset` and Transformation<jupyter_code>def one_hot_label(image, label):
  label = tf.one_hot(label, depth=10)
  return image, label
tf.set_random_seed(219)
batch_size = 32
max_epochs = 1

# for train
N = len(train_data)
train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))
train_dataset = train_dataset.shuffle(buffer_size=10000)
train_dataset = train_dataset.map(one_hot_label)
train_dataset = train_dataset.batch(batch_size=batch_size)
print(train_dataset)

# for test
test_dataset = tf.data.Dataset.from_tensor_slices((test_data, test_labels))
test_dataset = test_dataset.map(one_hot_label)
test_dataset = test_dataset.batch(batch_size=batch_size)
print(test_dataset)<jupyter_output><empty_output><jupyter_text>Create the model* Use `tf.keras.layers` * Use `tf.keras.Sequential()` API (01.mnist.LeNet5.ipynb)```pythonmodel = tf.keras.Sequential()model.add(layers.Conv2D(filters=32, kernel_size=[5, 5], padding='same', activation='relu'))model.add(layers.MaxPool2D())model.add(layers.Conv2D(filters=64, kernel_size=[5, 5], padding='same', activation='relu'))model.add(layers.MaxPool2D())model.add(layers.Flatten())model.add(layers.Dense(1024, activation='relu'))model.add(layers.Dense(10, activation='softmax'))```<jupyter_code>class MNISTModel(tf.keras.Model):
  def __init__(self):
    super(MNISTModel, self).__init__()
    self.conv1 = layers.Conv2D(filters=32, kernel_size=[5, 5], padding='same', activation='relu')
    self.pool1 = layers.MaxPool2D()
    self.conv2 = layers.Conv2D(filters=64, kernel_size=[5, 5], padding='same', activation='relu')
    self.pool2 = layers.MaxPool2D()
    self.flatten = layers.Flatten()
    self.dense1 = layers.Dense(units=1024, activation='relu')
    self.dense2 = layers.Dense(units=10, activation='softmax')

  def call(self, inputs):
    """Run the model."""
    conv1 = self.conv1(inputs)
    pool1 = self.pool1(conv1)
    conv2 = self.conv2(pool1)
    pool2 = self.pool2(conv2)
    flatten = self.flatten(pool2)
    dense1 = self.dense1(flatten)
    logits = self.dense2(dense1)
    
    return logits
model = MNISTModel()
# without training, just inference a model in eager execution:
for images, labels in train_dataset.take(1):
  predictions = model(images[0:1])
  print("Predictions: ", predictions.numpy())
model.summary()<jupyter_output><empty_output><jupyter_text>Train a model Define loss and accuray functions<jupyter_code>loss_object = tf.keras.losses.CategoricalCrossentropy()
acc_object = tf.keras.metrics.CategoricalAccuracy()<jupyter_output><empty_output><jupyter_text>Define a optimizer<jupyter_code># use Adam optimizer 
optimizer = tf.train.AdamOptimizer(1e-4)

# record loss for every epoch
mean_loss = tf.keras.metrics.Mean("loss")

# save loss and accuracy history for plot
loss_history = []
accuracy_history = [(0, 0.0)]<jupyter_output><empty_output><jupyter_text>Train a model<jupyter_code>print("start training!")
global_step = 0
num_batches_per_epoch = int(N / batch_size)

for epoch in range(max_epochs):
  
  for step, (images, labels) in enumerate(train_dataset):
    start_time = time.time()
    
    with tf.GradientTape() as tape:
      predictions = model(images)
      loss_value = loss_object(labels, predictions)
      acc_value = acc_object(labels, predictions)

    gradients = tape.gradient(loss_value, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    global_step += 1
    
    mean_loss(loss_value)
    loss_history.append((global_step, mean_loss.result().numpy()))
    
    if global_step % 10 == 0:
      clear_output(wait=True)
      epochs = epoch + step / float(num_batches_per_epoch)
      duration = time.time() - start_time
      examples_per_sec = batch_size / float(duration) 
      print("epochs: {:.2f}, step: {}, loss: {:.3g}, accuracy: {:.4g}% ({:.2f} examples/sec; {:.4f} sec/batch)".format(
          epochs, global_step, mean_loss.result().numpy(), acc_value.numpy()*100, examples_per_sec, duration))
      
  # save mean accuracy for plot
  accuracy_history.append((global_step, acc_value.numpy()*100))

  # clear the history
  mean_loss.reset_states()
  acc_object.reset_states()

print("training done!")<jupyter_output><empty_output><jupyter_text>Plot the loss funtion<jupyter_code>plt.plot(*zip(*loss_history), label='loss')
plt.xlabel('Number of steps')
plt.ylabel('Loss value [cross entropy]')
plt.legend()
plt.show()
plt.plot(*zip(*accuracy_history), 'bo', label='accuracy')
plt.xlabel('Number of steps')
plt.ylabel('Accuracy value')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Evaluate a model Test trained model* test accuracy: 0.9798 for 1 epochs<jupyter_code>acc_object.reset_states()

for images, labels in test_dataset:
  predictions = model(images)
  acc_object(labels, predictions)
  
print("test accuracy: {:.4g}%".format(acc_object.result() * 100))<jupyter_output><empty_output><jupyter_text>Plot test set<jupyter_code>np.random.seed(219)
test_batch_size = 16
batch_index = np.random.choice(len(test_data), size=test_batch_size, replace=False)

batch_xs = test_data[batch_index]
batch_ys = test_labels[batch_index]
y_pred_ = model(batch_xs)

fig = plt.figure(figsize=(16, 10))
for i, (px, py) in enumerate(zip(batch_xs, y_pred_)):
  p = fig.add_subplot(4, 8, i+1)
  if np.argmax(py) == batch_ys[i]:
    p.set_title("y_pred: {}".format(np.argmax(py)), color='blue')
  else:
    p.set_title("y_pred: {}".format(np.argmax(py)), color='red')
  p.imshow(px.reshape(28, 28))
  p.axis('off')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13448698732797384, 'fraction_numerical': 0.02152881864014171, 'mean_word_length': 6.133138969873664, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '316199', 'license': 'Apache-2.0', 'n_tokens_mistral': 2718, 'n_tokens_neox': 2584, 'n_words': 485, 'path': 'tf.version.2/03.cnn/02.mnist.LeNet5.functional.API.ipynb', 'repo_name': 'ilguyi/tensorflow.tutorials', 'size': '7339'}