<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;MNIST Dataset1.1&nbsp;&nbsp;Get the MNIST data1.2&nbsp;&nbsp;MLP in TensorFlow1.3&nbsp;&nbsp;MLP in Keras1.4&nbsp;&nbsp;MLP in TFLearn2&nbsp;&nbsp;TimeSeries Data - MLP - Keras2.1&nbsp;&nbsp;Prepare the data2.2&nbsp;&nbsp;Build, Train and Evaluate the Model  MultiLayer Perceptron<jupyter_code>import numpy as np
np.random.seed(123)
print("NumPy:{}".format(np.__version__))

import pandas as pd
print("Pandas:{}".format(pd.__version__))

import math
import os

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.pylab import rcParams
rcParams['figure.figsize'] = 15, 10
print("Matplotlib:{}".format(mpl.__version__))

import tensorflow as tf
tf.set_random_seed(123)
print("TensorFlow:{}".format(tf.__version__))

import keras
print("Keras:{}".format(keras.__version__))
DATASETSLIB_HOME = '../datasetslib'
import sys
if not DATASETSLIB_HOME in sys.path:
    sys.path.append(DATASETSLIB_HOME)
%reload_ext autoreload
%autoreload 2
import datasetslib

from datasetslib import util as dsu
datasetslib.datasets_root = os.path.join(os.path.expanduser('~'),'datasets')<jupyter_output><empty_output><jupyter_text>MNIST Dataset  Get the MNIST data<jupyter_code>from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(os.path.join(datasetslib.datasets_root, 'mnist'),
                                  one_hot=True)

X_train = mnist.train.images
X_test = mnist.test.images
Y_train = mnist.train.labels
Y_test = mnist.test.labels

num_outputs = 10  # 0-9 digits
num_inputs = 784  # total pixels<jupyter_output>Extracting /home/armando/datasets/mnist/train-images-idx3-ubyte.gz
Extracting /home/armando/datasets/mnist/train-labels-idx1-ubyte.gz
Extracting /home/armando/datasets/mnist/t10k-images-idx3-ubyte.gz
Extracting /home/armando/datasets/mnist/t10k-labels-idx1-ubyte.gz<jupyter_text>MLP in TensorFlow<jupyter_code>tf.reset_default_graph()
def mlp(x, num_inputs, num_outputs, num_layers, num_neurons):
    w = []
    b = []
    for i in range(num_layers):
        # weights
        w.append(tf.Variable(tf.random_normal(
            [num_inputs if i == 0 else num_neurons[i - 1],
             num_neurons[i]]),
            name="w_{0:04d}".format(i)
        ))
        # biases
        b.append(tf.Variable(tf.random_normal(
            [num_neurons[i]]),
            name="b_{0:04d}".format(i)
        ))
    w.append(tf.Variable(tf.random_normal(
        [num_neurons[num_layers - 1] if num_layers > 0 else num_inputs,
         num_outputs]), name="w_out"))
    b.append(tf.Variable(tf.random_normal([num_outputs]), name="b_out"))

    # x is input layer
    layer = x
    # add hidden layers
    for i in range(num_layers):
        layer = tf.nn.relu(tf.matmul(layer, w[i]) + b[i])
    # add output layer
    layer = tf.matmul(layer, w[num_layers]) + b[num_layers]

    return layer


def mnist_batch_func(batch_size=100):
    X_batch, Y_batch = mnist.train.next_batch(batch_size)
    return [X_batch, Y_batch]


def tensorflow_classification(n_epochs, n_batches,
                              batch_size, batch_func,
                              model, optimizer, loss, accuracy_function,
                              X_test, Y_test):
    with tf.Session() as tfs:
        tfs.run(tf.global_variables_initializer())
        for epoch in range(n_epochs):
            epoch_loss = 0.0
            for batch in range(n_batches):
                X_batch, Y_batch = batch_func(batch_size)
                feed_dict = {x: X_batch, y: Y_batch}
                _, batch_loss = tfs.run([optimizer, loss], feed_dict)
                epoch_loss += batch_loss
            average_loss = epoch_loss / n_batches
            print("epoch: {0:04d}   loss = {1:0.6f}".format(
                epoch, average_loss))
        feed_dict = {x: X_test, y: Y_test}
        accuracy_score = tfs.run(accuracy_function, feed_dict=feed_dict)
        print("accuracy={0:.8f}".format(accuracy_score))
tf.reset_default_graph()
num_layers = 0
num_neurons = []
learning_rate = 0.01
n_epochs = 50
batch_size = 100
n_batches = int(mnist.train.num_examples / batch_size)

# input images
x = tf.placeholder(dtype=tf.float32, name="x", shape=[None, num_inputs])
# target output
y = tf.placeholder(dtype=tf.float32, name="y", shape=[None, num_outputs])

model = mlp(x=x,
            num_inputs=num_inputs,
            num_outputs=num_outputs,
            num_layers=num_layers,
            num_neurons=num_neurons)

# loss function
loss = tf.reduce_mean(
    tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=y))
# optimizer function
optimizer = tf.train.GradientDescentOptimizer(
    learning_rate=learning_rate).minimize(loss)

predictions_check = tf.equal(tf.argmax(model, 1), tf.argmax(y, 1))
accuracy_function = tf.reduce_mean(tf.cast(predictions_check, tf.float32))

tensorflow_classification(n_epochs=n_epochs,
                          n_batches=n_batches,
                          batch_size=batch_size,
                          batch_func=mnist_batch_func,
                          model=model,
                          optimizer=optimizer,
                          loss=loss,
                          accuracy_function=accuracy_function,
                          X_test=mnist.test.images,
                          Y_test=mnist.test.labels
                          )
tf.reset_default_graph()
num_layers = 1 
num_neurons = []
for i in range(num_layers):
    num_neurons.append(8)
    
learning_rate = 0.01
n_epochs = 50
batch_size = 100
n_batches = int(mnist.train.num_examples/batch_size)

# input images
x = tf.placeholder(dtype=tf.float32, name="x", shape=[None, num_inputs]) 
# target output
y = tf.placeholder(dtype=tf.float32, name="y", shape=[None, num_outputs]) 

model = mlp(x=x, 
            num_inputs=num_inputs, 
            num_outputs=num_outputs, 
            num_layers=num_layers, 
            num_neurons=num_neurons)

# loss function
#loss = tf.reduce_mean(-tf.reduce_sum(y * tf.log(model), axis=1))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=y))
# optimizer function
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss)
predictions_check = tf.equal(tf.argmax(model,1), tf.argmax(y,1))
accuracy_function = tf.reduce_mean(tf.cast(predictions_check, tf.float32))

tensorflow_classification(n_epochs=n_epochs, 
                          n_batches=n_batches, 
                          batch_size=batch_size, 
                          batch_func=mnist_batch_func, 
                          model = model, 
                          optimizer = optimizer, 
                          loss = loss, 
                          accuracy_function = accuracy_function, 
                          X_test = mnist.test.images, 
                          Y_test = mnist.test.labels
                         )
tf.reset_default_graph()
num_layers = 2
num_neurons = []
for i in range(num_layers):
    num_neurons.append(256)
    
learning_rate = 0.01
n_epochs = 50
batch_size = 100
n_batches = int(mnist.train.num_examples/batch_size)

# input images
x = tf.placeholder(dtype=tf.float32, name="x", shape=[None, num_inputs]) 
# target output
y = tf.placeholder(dtype=tf.float32, name="y", shape=[None, num_outputs]) 

model = mlp(x=x, 
            num_inputs=num_inputs, 
            num_outputs=num_outputs, 
            num_layers=num_layers, 
            num_neurons=num_neurons)

# loss function
#loss = tf.reduce_mean(-tf.reduce_sum(y * tf.log(model), axis=1))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model, 
                                                              labels=y))
# optimizer function
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss)

predictions_check = tf.equal(tf.argmax(model,1), tf.argmax(y,1))
accuracy_function = tf.reduce_mean(tf.cast(predictions_check, tf.float32))

tensorflow_classification(n_epochs=n_epochs, 
                          n_batches=n_batches, 
                          batch_size=batch_size, 
                          batch_func=mnist_batch_func, 
                          model = model, 
                          optimizer = optimizer, 
                          loss = loss, 
                          accuracy_function = accuracy_function, 
                          X_test = mnist.test.images, 
                          Y_test = mnist.test.labels
                         )<jupyter_output>epoch: 0000   loss = 57.009780
epoch: 0001   loss = 10.813600
epoch: 0002   loss = 6.615101
epoch: 0003   loss = 4.692290
epoch: 0004   loss = 3.588678
epoch: 0005   loss = 2.884803
epoch: 0006   loss = 2.387677
epoch: 0007   loss = 1.995401
epoch: 0008   loss = 1.740001
epoch: 0009   loss = 1.499358
epoch: 0010   loss = 1.322210
epoch: 0011   loss = 1.172742
epoch: 0012   loss = 1.049592
epoch: 0013   loss = 0.942527
epoch: 0014   loss = 0.862378
epoch: 0015   loss = 0.787306
epoch: 0016   loss = 0.707988
epoch: 0017   loss = 0.662152
epoch: 0018   loss = 0.607828
epoch: 0019   loss = 0.567524
epoch: 0020   loss = 0.524560
epoch: 0021   loss = 0.497883
epoch: 0022   loss = 0.457042
epoch: 0023   loss = 0.427090
epoch: 0024   loss = 0.401433
epoch: 0025   loss = 0.380523
epoch: 0026   loss = 0.362819
epoch: 0027   loss = 0.336290
epoch: 0028   loss = 0.329078
epoch: 0029   loss = 0.305773
epoch: 0030   loss = 0.299001
epoch: 0031   loss = 0.281987
epoch: 0032   loss = 0.271549
epoch: 0[...]<jupyter_text>MLP in Keras<jupyter_code>import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
tf.reset_default_graph()
keras.backend.clear_session()
num_layers = 2
num_neurons = []
for i in range(num_layers):
    num_neurons.append(256)
    
learning_rate = 0.01
n_epochs = 50
batch_size = 100

model = Sequential()
model.add(Dense(units=num_neurons[0], activation='relu', 
                input_shape=(num_inputs,)))
model.add(Dense(units=num_neurons[1], activation='relu'))
model.add(Dense(units=num_outputs, activation='softmax'))
model.summary()

model.compile(loss='categorical_crossentropy',
              optimizer=SGD(lr=learning_rate),
              metrics=['accuracy'])

model.fit(X_train, Y_train,
                    batch_size=batch_size,
                    epochs=n_epochs)

score = model.evaluate(X_test, Y_test)
print('\nTest loss:', score[0])
print('Test accuracy:', score[1])<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_1 (Dense)              (None, 256)               200960    
_________________________________________________________________
dense_2 (Dense)              (None, 256)               65792     
_________________________________________________________________
dense_3 (Dense)              (None, 10)                2570      
=================================================================
Total params: 269,322
Trainable params: 269,322
Non-trainable params: 0
_________________________________________________________________
Epoch 1/50
55000/55000 [==============================] - 2s 33us/step - loss: 1.0688 - acc: 0.7467
Epoch 2/50
55000/55000 [==============================] - 1s 23us/step - loss: 0.4358 - acc: 0.8813
Epoch 3/50
55000/55000 [==============================] - 1s 27us/st[...]<jupyter_text>MLP in TFLearn<jupyter_code>import tflearn
tf.reset_default_graph()
num_layers = 2
num_neurons = []
for i in range(num_layers):
    num_neurons.append(256)
    
learning_rate = 0.01
n_epochs = 50
batch_size = 100

# Build deep neural network
input_layer = tflearn.input_data(shape=[None, num_inputs])
dense1 = tflearn.fully_connected(input_layer, num_neurons[0], activation='relu')
dense2 = tflearn.fully_connected(dense1, num_neurons[1], activation='relu')
softmax = tflearn.fully_connected(dense2, num_outputs, activation='softmax')

optimizer = tflearn.SGD(learning_rate=learning_rate)
net = tflearn.regression(softmax, optimizer=optimizer, 
                         metric=tflearn.metrics.Accuracy(), 
                         loss='categorical_crossentropy')
model = tflearn.DNN(net)

model.fit(X_train, Y_train, 
          n_epoch=n_epochs, batch_size=batch_size, 
          show_metric=True, run_id='dense_model')

score = model.evaluate(X_test, Y_test)
print('Test accuracy:', score[0])<jupyter_output>Training Step: 27499  | total loss: [1m[32m0.10358[0m[0m | time: 2.698s
| SGD | epoch: 050 | loss: 0.10358 - acc: 0.9742 -- iter: 54900/55000
Training Step: 27500  | total loss: [1m[32m0.10037[0m[0m | time: 2.702s
| SGD | epoch: 050 | loss: 0.10037 - acc: 0.9738 -- iter: 55000/55000
--
Test accuracy: 0.9657<jupyter_text>TimeSeries Data - MLP - Keras  Prepare the data<jupyter_code>import os

dataframe = pd.read_csv(os.path.join(datasetslib.datasets_root, 
                                     'ts-data', 
                                     'international-airline-passengers-cleaned.csv'), 
                        usecols=[1],header=0)
dataset = dataframe.values
dataset = dataset.astype('float32')
# normalize the dataset
#scaler = skpp.MinMaxScaler(feature_range=(0, 1))
#normalized_dataset = scaler.fit_transform(dataset)
# split into train and test sets
train,test=dsu.train_test_split(dataset,train_size=0.67)
print(len(train), len(test))
# reshape into X=t-1,t and Y=t+1
n_x=2
n_y=1

X_train, Y_train, X_test, Y_test = dsu.mvts_to_xy(train,test,n_x=n_x,n_y=n_y)<jupyter_output><empty_output><jupyter_text>Build, Train and Evaluate the Model<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
tf.reset_default_graph()
keras.backend.clear_session()
num_layers = 2
num_neurons = [8,8]
n_epochs = 50
batch_size = 2

model = Sequential()
model.add(Dense(num_neurons[0], activation='relu', input_shape=(n_x,)))
model.add(Dense(num_neurons[1], activation='relu'))
model.add(Dense(units=1))
model.summary()

model.compile(loss='mse', optimizer='adam')

model.fit(X_train, Y_train,
                    batch_size=batch_size,
                    epochs=n_epochs)

score = model.evaluate(X_test, Y_test)
print('\nTest mse:', score)
print('Test rmse:', math.sqrt(score))
# make predictions
Y_train_pred = model.predict(X_train)
Y_test_pred = model.predict(X_test)

# shift train predictions for plotting
Y_train_pred_plot = np.empty_like(dataset)
Y_train_pred_plot[:, :] = np.nan
Y_train_pred_plot[n_x-1:len(Y_train_pred)+n_x-1, :] = Y_train_pred

# shift test predictions for plotting
Y_test_pred_plot = np.empty_like(dataset)
Y_test_pred_plot[:, :] = np.nan
Y_test_pred_plot[len(Y_train_pred)+(n_x*2)-1:len(dataset)-1, :] = Y_test_pred

# plot baseline and predictions
plt.plot(dataset,label='Original Data')
plt.plot(Y_train_pred_plot,label='Y_train_pred')
plt.plot(Y_test_pred_plot,label='Y_test_pred')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12503325352487363, 'fraction_numerical': 0.04974727321096036, 'mean_word_length': 3.1378646119977986, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '227562', 'license': 'MIT', 'n_tokens_mistral': 5674, 'n_tokens_neox': 5145, 'n_words': 932, 'path': 'Chapter05/ch-05_MLP.ipynb', 'repo_name': 'vgoklani/Mastering-TensorFlow-1x', 'size': '15036'}