<jupyter_start><jupyter_text>MNIST classification example with TensorFlow  Install packages on Google  Cloud Datalab (locally use conda env)  Select in the Python3 Kernel:In the menu bar the of 'Kernel', select   **python3** Install needed packagescopy the command below in a Google Cloud Datalab cell  **!pip install tensorflow==1.12** Restart the Kernel this is to take into account the new installed packages. Click in the menu bar on:  **Reset Session**  Include paths to our functions<jupyter_code>import sys
import os
import pathlib

workingdir=os.getcwd()
print(workingdir)
d=[d for d in os.listdir(workingdir)]
n=0
while not set(['notebook']).issubset(set(d)):
    workingdir=str(pathlib.Path(workingdir).parents[0])
    print(workingdir)
    d=[d for d in os.listdir(str(workingdir))]
    n+=1
    if n>5:
        break
sys.path.insert(0, workingdir)
os.chdir(workingdir)<jupyter_output>/Users/tarrade/Desktop/Work/Data_Science/Tutorials_Codes/Python/proj_DL_models_and_pipelines_with_GCP/notebook
/Users/tarrade/Desktop/Work/Data_Science/Tutorials_Codes/Python/proj_DL_models_and_pipelines_with_GCP<jupyter_text>Setup librairies import and plots style  Import librairies<jupyter_code>import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import _pickle as cPickle
tf.__version__<jupyter_output><empty_output><jupyter_text>Import our utils functions<jupyter_code>import src.utils.mnist_utils as mnist_utils
import importlib
importlib.reload(mnist_utils);# to reload the function and mask the output<jupyter_output><empty_output><jupyter_text>Set plots style<jupyter_code>print(plt.style.available)
# color code: https://matplotlib.org/gallery/color/named_colors.html#sphx-glr-gallery-color-named-colors-py
plt.style.use('seaborn-ticks')<jupyter_output><empty_output><jupyter_text>Data visualization of some examples per classes  Loading dataset<jupyter_code>path_train='data/mnist/numpy_train/'
path_test='data/mnist/numpy_test/'
x_train=cPickle.load(open(path_train+'x_train.pkl', 'rb'))
y_train=cPickle.load(open(path_train+'y_train.pkl', 'rb'))
x_test=cPickle.load(open(path_test+'x_test.pkl', 'rb'))
y_test=cPickle.load(open(path_test+'y_test.pkl', 'rb'))<jupyter_output><empty_output><jupyter_text>Looking at some example from training dataset<jupyter_code>mnist_utils.plot_mnist_images(x_train[0:25], y_train[0:25])<jupyter_output><empty_output><jupyter_text>Data exploration<jupyter_code># check data shape (training)
x_train.shape
# check data shape (train)
x_test.shape
x_train.dtype, x_test.dtype
np.max(x_train), np.min(x_train), np.max(x_test), np.min(x_test)<jupyter_output><empty_output><jupyter_text>Normalize and reorganize the data<jupyter_code># cast uint8 -> float32
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
# renormalize the data 255 grey variation
x_train /= 255
x_test /= 255
# reshape the data 28 x 28 -> 784
x_train = x_train.reshape(len(x_train), x_train.shape[1]*x_train.shape[2])
x_test = x_test.reshape(len(x_test), x_test.shape[1]*x_test.shape[2])
x_train.shape
x_test.shape<jupyter_output><empty_output><jupyter_text>Reshape the labels<jupyter_code>y_train.shape
y_test.shape
np.unique(y_train), np.unique(y_test)
num_classes = len(np.unique(y_train))
num_classes
y_train_ = np.zeros((len(y_train), num_classes))
y_test_ = np.zeros((len(y_test), num_classes))
y_train_[np.arange(len(y_train)), y_train] = 1
y_test_[np.arange(len(y_test)), y_test] = 1
y_train=y_train_
y_test=y_test_
y_train.shape
y_test.shape
dim_input=x_train.shape[1]
dim_input<jupyter_output><empty_output><jupyter_text>Defined some hyperparameters<jupyter_code># print info every n_print
n_print = 1

# hidden layer 1
n1=300<jupyter_output><empty_output><jupyter_text>Defined our model<jupyter_code># x [60000, 784]
# y [60000, 10]

# 1 layer n1 with 200 neurones
# n1 = 300

#  . . . . . .    input data (ffattened pixels)      x [batch    ,   dim_input]
#   \x/\x/x/      -- fully connected layer (relu)    W1[dim_input,          n1]      B1[n1         ]
#    . . .                                           Y1[batch    ,          n1]
#     \x/         -- fully connected layer (softmax) W2[n1       , num_classes]      B2[num_classes]
#      .                                             Y2[batch    , num_classes]

# reset graph before starting
tf.reset_default_graph()

# tensor (placeholder) for the learning rate
learning_rate = tf.placeholder(tf.float32, shape=())

# tensor (placeholder) for the input data [60000, 784]
x = tf.placeholder(tf.float32, [None, dim_input])

# tensor (placeholder) for the output data [60000, 10]
y = tf.placeholder(tf.float32, [None, num_classes])

# now declare the weights connecting the input to the hidden layer
W1 = tf.Variable(tf.random_normal([dim_input, n1], stddev=0.03), name='W1')
b1 = tf.Variable(tf.random_normal([n1]), name='b1')
# and the weights connecting the hidden layer to the output layer
W2 = tf.Variable(tf.random_normal([n1, num_classes], stddev=0.03), name='W2')
b2 = tf.Variable(tf.random_normal([num_classes]), name='b2')

# initialization for W
#tf.Variable(tf.random_normal([], stddev = stddev))
#tf.Variable(tf.truncated_normal([], stddev = stddev))

# initialization for b
#tf.Variable(tf.constant(0.0, shape = []))
#tf.Variable(tf.random_normal([], stddev = stddev))

# calculate the output of the hidden layer
Y1 = tf.nn.relu(tf.matmul(x, W1) + b1)

# last layer
Ylogits= tf.matmul(Y1, W2) + b2

# output layer
y_ = tf.nn.softmax(Ylogits)

# now let's define the cost function which we are going to train the model on

# cross entropy defined manually
# -sum(y * log(y_) + (1-y) * log(1-y_))
y_clipped = tf.clip_by_value(y_, 1e-10, 0.9999999) # if value are below 1e-10 return 1e-10 same of 0.9999999
cross_entropy = tf.reduce_sum(y * tf.log(y_clipped)+ (1 - y) * tf.log(1 - y_clipped), axis=1)
cost = -tf.reduce_mean(cross_entropy)

# add an optimiser
optimiser = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)

# finally setup the initialisation operator
init= tf.global_variables_initializer()

# define an accuracy assessment operation
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
def run_logistic_model(learning_r, training_epochs, train_obs, train_labels, test_obs, test_labels, debug = False):
    sess = tf.Session()
    sess.run(init)
    
    cost_history_train = np.empty(shape=[0], dtype = float)
    cost_history_test = np.empty(shape=[0], dtype = float)

    for epoch in range(training_epochs+1):
        
        sess.run(optimiser, feed_dict = {x: train_obs, y: train_labels, learning_rate: learning_r})

        cost_train = sess.run(cost, feed_dict={x: train_obs, y: train_labels, learning_rate: learning_r})
        cost_history_train = np.append(cost_history_train, cost_train)
        
        cost_test = sess.run(cost, feed_dict={x: test_obs, y: test_labels, learning_rate: learning_r})
        cost_history_test = np.append(cost_history_test, cost_test)
        
        
        if (epoch % n_print == 0) & debug:
            print("Reached epoch",epoch,"cost J =", str.format('{0:.6f}', cost_train))
            
            acc_train = sess.run(accuracy, feed_dict={x: train_obs, y: train_labels})
            acc_test = sess.run(accuracy, feed_dict={x: test_obs, y: test_labels})
            print(" accurary on the training set", str.format('{0:.2f}', acc_train))
            print(" accurary on the testing set", str.format('{0:.2f}', acc_test))
            
    return sess, cost_history_train, cost_history_test
sess, cost_history_train, cost_history_test = run_logistic_model(learning_r = 0.2, 
                                                                 training_epochs = 50, 
                                                                 train_obs = x_train, 
                                                                 train_labels = y_train, 
                                                                 test_obs = x_test, 
                                                                 test_labels = y_test, 
                                                                 debug = True)
print ("Accuracy on training data:", accuracy.eval(feed_dict={x: x_train, y: y_train}, session = sess))
print ("Accuracy on testing data:", accuracy.eval(feed_dict={x: x_test, y: y_test}, session = sess))
        
plt.rc('xtick', labelsize='x-small')
plt.rc('ytick', labelsize='x-small')

fig = plt.figure(figsize=(10,8))
ax = fig.add_subplot(1, 1, 1)
ax.plot(cost_history_train, ls='-', color = 'black', lw = 3, label = r'Training $\gamma = 0.01$')
ax.plot(cost_history_test, ls='--', color = 'blue', lw = 3, label = r'Testing $\gamma = 0.01$')

ax.set_xlabel('epochs', fontsize = 16)
ax.set_ylabel('Cost function $J$', fontsize = 16)

plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0., fontsize = 16)
plt.tick_params(labelsize=16);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12363433667781494, 'fraction_numerical': 0.02318840579710145, 'mean_word_length': 3.807609860664523, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '577252', 'license': 'Apache-2.0', 'n_tokens_mistral': 3265, 'n_tokens_neox': 3111, 'n_words': 670, 'path': 'notebook/01-Example_mnist_tensorflow_low_level.ipynb', 'repo_name': 'tarrade/proj_DL_models_and_pipelines_with_GCP', 'size': '8970'}