<jupyter_start><jupyter_text>Classify handwritten digits with Keras Data from: [the MNIST dataset](http://yann.lecun.com/exdb/mnist/) 1. [Download the MNIST dataset from Internet](01)2. [Preprocessing the dataset](02)3. [Softmax Regression](03)4. [A small Convolutional Neural Network](04) ---<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set()
import pandas as pd
import sklearn
import os
import requests
from tqdm._tqdm_notebook import tqdm_notebook
import tarfile<jupyter_output><empty_output><jupyter_text>1. Download the MNIST dataset from Internet  I've made the dataset into a zipped tar file. You'll have to download it now.<jupyter_code>def download_file(url,file):
    # Streaming, so we can iterate over the response.
    r = requests.get(url, stream=True)

    # Total size in bytes.
    total_size = int(r.headers.get('content-length', 0)); 
    block_size = 1024
    wrote = 0 
    with open(file, 'wb') as f:
        for data in tqdm_notebook(r.iter_content(block_size), total=np.ceil(total_size//block_size) , unit='KB', unit_scale=True):
            wrote = wrote  + len(data)
            f.write(data)
    if total_size != 0 and wrote != total_size:
        print("ERROR, something went wrong") 

url = "https://github.com/chi-hung/PythonTutorial/raw/master/datasets/mnist.tar.gz"
file = "mnist.tar.gz"
print('Retrieving the MNIST dataset...')
download_file(url,file)
print('Extracting the MNIST dataset...')
tar = tarfile.open(file)
tar.extractall()
tar.close()
print('Completed fetching the MNIST dataset.')<jupyter_output><empty_output><jupyter_text>10 folders of images will be extracted from the downloaded tar file. ---  2. Preprocessing the dataset<jupyter_code>def filePathsGen(rootPath):
    paths=[]
    dirs=[]
    for dirPath,dirNames,fileNames in os.walk(rootPath):
        for fileName in fileNames:
            fullPath=os.path.join(dirPath,fileName)
            paths.append((int(dirPath[len(rootPath) ]),fullPath))
        dirs.append(dirNames)
    return dirs,paths
dirs,paths=filePathsGen('mnist/') # load the image paths
dfPath=pd.DataFrame(paths,columns=['class','path']) # save image paths as a Pandas DataFrame
dfPath.head(5) # see the first 5 paths of the DataFrame<jupyter_output><empty_output><jupyter_text>How many digit classes & how many figures belong to each of the classes?<jupyter_code>dfCountPerClass=dfPath.groupby('class').count()
dfCountPerClass.rename(columns={'path':'amount of figures'},inplace=True)
dfCountPerClass.plot(kind='bar',rot=0)<jupyter_output><empty_output><jupyter_text>Split the image paths into train($70\%$), val($15\%$), test($15\%$)<jupyter_code>train=dfPath.sample(frac=0.7) # sample 70% data to be the train dataset
test=dfPath.drop(train.index) # the rest 30% are now the test dataset

# take 50% of the test dataset as the validation dataset
val=test.sample(frac=1/2)
test=test.drop(val.index)

# let's check the length of the train, val and test dataset.
print('number of all figures = {:10}.'.format(len(dfPath)))
print('number of train figures= {:9}.'.format(len(train)))
print('number of val figures= {:10}.'.format(len(val)))
print('number of test figures= {:9}.'.format(len(test)))

# let's take a look: plotting 3 figures from the train dataset
for j in range(3):
    img=plt.imread(train['path'].iloc[j])
    plt.imshow(img,cmap="gray")
    plt.axis("off")
    plt.show()<jupyter_output><empty_output><jupyter_text>Load images into RAM<jupyter_code>def dataLoad(dfPath):
    paths=dfPath['path'].values
    x=np.zeros((len(paths),28,28),dtype=np.float32 )

    for j in range(len(paths)):
        x[j,:,:]=plt.imread(paths[j])/255

    y=dfPath['class'].values
    return x,y
train_x,train_y=dataLoad(train)
val_x,val_y=dataLoad(val)
test_x,test_y=dataLoad(test)<jupyter_output><empty_output><jupyter_text>Remark: loading all images to RAM might take a while.<jupyter_code>print("tensor shapes:\n")
print('train:',train_x.shape,train_y.shape)
print('val  :',val_x.shape,val_y.shape)
print('test :',test_x.shape,test_y.shape)<jupyter_output><empty_output><jupyter_text>---  3. Softmax Regression<jupyter_code>from keras.models import Sequential
from keras.layers import Dense,Flatten
from keras.optimizers import SGD<jupyter_output><empty_output><jupyter_text>Onehot-encoding the labels:<jupyter_code>from sklearn.preprocessing import OneHotEncoder
enc = OneHotEncoder()
train_y_onehot = np.float32( enc.fit_transform(train_y.reshape(-1,1)) \
                   .toarray() )

val_y_onehot = np.float32( enc.fit_transform(val_y.reshape(-1,1)) \
                 .toarray() )

test_y_onehot = np.float32( enc.fit_transform(test_y.reshape(-1,1)) \
                  .toarray() )<jupyter_output><empty_output><jupyter_text>Construct the model:<jupyter_code>model = Sequential()
model.add(Flatten(input_shape=(28,28)))
model.add(Dense(10, activation='softmax') )

sgd=SGD(lr=0.2, momentum=0.0, decay=0.0)
model.compile(optimizer='sgd',
      loss='categorical_crossentropy',
      metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>More details about the constructed model:<jupyter_code>model.summary()<jupyter_output><empty_output><jupyter_text>Train the model:<jupyter_code>hist=model.fit(train_x, train_y_onehot,
               epochs=20, batch_size=128,
               validation_data=(val_x,val_y_onehot))<jupyter_output><empty_output><jupyter_text>See how the accuracy climbs during training:<jupyter_code>plt.plot(hist.history['acc'],ms=5,marker='o',label='accuracy')
plt.plot(hist.history['val_acc'],ms=5,marker='o',label='val accuracy')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Now, you'll probably want to evaluate or save the trained model.<jupyter_code># calculate loss & accuracy (evaluated on the test dataset)
score = model.evaluate(test_x, test_y_onehot, batch_size=128)
print("LOSS (evaluated on the test dataset)=     {}".format(score[0]))
print("ACCURACY (evaluated on the test dataset)= {}".format(score[1]))<jupyter_output><empty_output><jupyter_text>Save model architecture & weights:<jupyter_code>import json
with open('first_try.json', 'w') as jsOut:
    json.dump(model.to_json(), jsOut)

model.save_weights('first_try.h5')<jupyter_output><empty_output><jupyter_text>Load the saved model architecture & weights:<jupyter_code>from keras.models import model_from_json
with open('first_try.json', 'r') as jsIn:
    model_architecture=json.load(jsIn)
    
model_new=model_from_json(model_architecture)
model_new.load_weights('first_try.h5')
model_new.summary()<jupyter_output><empty_output><jupyter_text>Output the classification report (see if the trained model works well on the test data):<jupyter_code>pred_y=model.predict(test_x).argmax(axis=1)
from sklearn.metrics import classification_report
print( classification_report(test_y,pred_y) )<jupyter_output><empty_output><jupyter_text>4. A small Convolutional Neural Network Reshape the tensors (this step is necessary, because the CNN model wants the input tensor to be 4D):<jupyter_code>train_x = np.expand_dims(train_x,axis=-1)
val_x = np.expand_dims(val_x,axis=-1)
test_x = np.expand_dims(test_x,axis=-1)<jupyter_output><empty_output><jupyter_text>Create the model:<jupyter_code>from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten,Conv2D, MaxPooling2D
from keras.layers import Activation
from keras.optimizers import SGD

in_shape=(28,28,1)
# ========== BEGIN TO CREATE THE MODEL ==========
model = Sequential()
# feature extraction (2 conv layers)
model.add(Conv2D(32, (3,3),
                 activation='relu',
                 input_shape=in_shape))
model.add(Conv2D(64, (3,3), activation='relu')
         )
model.add(MaxPooling2D(pool_size=(2, 2))
         )
model.add(Dropout(0.5))
model.add(Flatten())
# classification (2 dense layers)
model.add(Dense(128, activation='relu')
         )
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
# ========== COMPLETED THE MODEL CREATION========

# Compile the model before training.
model.compile(loss='categorical_crossentropy',
              optimizer=SGD(lr=0.01,momentum=0.1),
              metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Train the model:<jupyter_code>%%time
hist=model.fit(train_x, train_y_onehot, 
               epochs=20,
               batch_size=32,
               validation_data=(val_x,val_y_onehot),
              )<jupyter_output><empty_output><jupyter_text>See how the accuracy climbs during training:<jupyter_code>plt.plot(hist.history['acc'],ms=5,marker='o',label='accuracy')
plt.plot(hist.history['val_acc'],ms=5,marker='o',label='val accuracy')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Output the classification report (see if the trained model works well on the test data):<jupyter_code>pred_y=model.predict(test_x).argmax(axis=1)
from sklearn.metrics import classification_report
print( classification_report(test_y,pred_y) )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13668185353928214, 'fraction_numerical': 0.016335148349816645, 'mean_word_length': 5.721433905899925, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 89, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '392046', 'license': 'MIT', 'n_tokens_mistral': 3225, 'n_tokens_neox': 3110, 'n_words': 623, 'path': 'code_examples/KerasMNISTDemo.ipynb', 'repo_name': 'chi-hung/PythonTutorial', 'size': '8999'}