<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# General tools
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from tqdm import tqdm

# Metrics and Evaluation
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn import metrics
import sklearn.metrics as met

# KNearest Neighbours
from sklearn.cluster import KMeans
from sklearn.neighbors import KNeighborsClassifier  

# SVC
from sklearn.svm import  SVC
from sklearn.model_selection import validation_curve
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

# Decision Tree
from sklearn import tree
from sklearn.tree import  DecisionTreeClassifier, export_graphviz
import subprocess
from sklearn.ensemble import RandomForestClassifier

# Deep Neural Network
from keras.models import Model
from keras.layers import Input, Conv2D, MaxPool2D, Flatten, Concatenate, Dense, Dropout, BatchNormalization
from keras.optimizers import Adam  # First, naive approach
from keras.losses import categorical_crossentropy
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, CSVLogger  # Add callbacks to model
from keras.models import load_model  # Save model params
from keras.layers import LeakyReLU, concatenate
from keras.layers.advanced_activations import PReLU
from keras.initializers import glorot_normal
from keras.utils import to_categorical
from keras.utils import Sequence
from sklearn.utils import shuffle
import keras.backend as K<jupyter_output>Using TensorFlow backend.<jupyter_text>Load data<jupyter_code>def extract_dataset(path):
  """
  extract DoubledMNIST dataset
  Argument: path to .zip file with the dataset
  Return value: x_train, y_train, x_test, y_test lists of numpy arrays 
  
  (DoubledMNIST dataset: train size 120k images 56x56, test size 20k images 56x56)
  """
  # import libraries
  import os                     # for basic os operations
  from zipfile import ZipFile 
  from skimage import io
  import numpy as np
  import shutil
  
  if not path.endswith('.zip'):
    raise ValueError("Error: path is not '.zip' file")
  
  archive = ZipFile(path, 'r')  # extract
  archive.extractall('./DoubledMNIST')
  archive.close()
  del archive
  
  x_train = []
  y_train = []
  x_test = []
  y_test = []
  
  for file in os.listdir('./DoubledMNIST/train'):
    img = io.imread(os.path.join('./DoubledMNIST/train', file))
    x_train.append(np.array(img))
    y_train.append(int(file.split('_')[1]))
  
  for file in os.listdir('./DoubledMNIST/test'):
    img = io.imread(os.path.join('./DoubledMNIST/test', file))
    x_test.append(np.array(img))
    y_test.append(int(file.split('_')[1]))
    
  shutil.rmtree('./DoubledMNIST')
  return x_train, y_train, x_test, y_test
%%time
x_train, y_train, x_test, y_test = extract_dataset('./drive/My Drive/dm_sem/DoubledMNIST.zip')
print('No of train images: ', len(x_train))  # size check
print('No of train labels: ', len(y_train))
print('No of test images: ', len(x_test))
print('No of test labels: ', len(y_test))
x_train[0]  # image check; its saved as 8bit grayscale image
# Plot some examples from datbase
print('Slucajni uzorak baze podataka: ')
fig, ax = plt.subplots(nrows=4, ncols=10)
fig.set_figheight(9)
fig.set_figwidth(18)
for row in ax:
    for col in row:
      i = np.random.randint(len(x_train))
      col.imshow(x_train[i], cmap='gray')
      
plt.show()
x_train[0].shape  # check image shape
type(x_train[0])  # check image type<jupyter_output><empty_output><jupyter_text>Data for Choosing Parameters / Final Train<jupyter_code># Data for evaluation
x_train_full = np.array([img.flatten()/255.0 for img in x_train])  # image preprocessiong, convert it to [0, 1] and flatten for knn; knn dont requests normalize data?
y_train_full = np.array(y_train)

x_test_full = np.array([img.flatten()/255.0 for img in x_test])
y_test_full = np.array(y_test)
# Data for parameter choosing (knn, svm, dt)
x_train_choose = np.array([img.flatten()/255.0 for img in x_train[:1000]])  
y_train_choose = np.array(y_train[:1000])
x_test_choose = np.array([img.flatten()/255.0 for img in x_train[1000:2000]])  
y_test_choose = np.array(y_train[1000:2000])
plt.figure(figsize=(10,10))
print('Raspodela labela na celom skupu podataka')
plt.pie([sum(y_train_full==y) for y in range(10)], shadow=True, startangle=90, labels = ['0','1','2','3','4','5','6','7','8','9'], autopct='%1.1f%%')
plt.legend()
plt.show()
plt.figure(figsize=(10,10))
print('Raspodela labela na skupu za izbor parametara')
plt.pie([sum(y_train_choose==y) for y in range(10)], shadow=True, startangle=90, labels = ['0','1','2','3','4','5','6','7','8','9'], autopct='%1.1f%%')
plt.legend()
plt.show()
labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
# print concrete sample for parameter choosing
sns.countplot(y_train_choose)
# print concrete sample for parameter choosing
sns.countplot(y_test_choose)
# print basic statistics for any of the 56x56 pixels
data = pd.DataFrame(np.array(x_train_choose).reshape(len(x_train_choose), -1))
data.describe()<jupyter_output><empty_output><jupyter_text>KNearest Neighbours  Choose parameters<jupyter_code>%%time

ks = [1,2,3,4,5,6,7,8,9,10,30]  # choose appropriate k
ps = [1, 2]
ws = ['uniform', 'distance']

max_params = {'k':0, 'p':0, 'w':0}
max_score = 0
for k in ks:
  for p in ps:
    for w in ws:
      knn = KNeighborsClassifier(n_neighbors=k, p=p, weights=w)  # approx ~ 1nn
      knn.fit(x_train_choose, y_train_choose)
  
      score = accuracy_score(y_test_choose, knn.predict(x_test_choose))  # check acc score
      print(f"knn(k={k}, p={p}, weights={w}) : {score}")
      if score>max_score:  # Okamov zilet
        max_params['k'] = k
        max_params['p'] = p
        max_params['w'] = w
        max_score = score
      del knn
print(max_params)
print(max_score)<jupyter_output>{'k': 3, 'p': 2, 'w': 'distance'}
0.894<jupyter_text>Evaluate on best params<jupyter_code>k = max_params['k']
p = max_params['p']
w = max_params['w']
%%time

knn = KNeighborsClassifier(n_neighbors=k, p=p, weights=w)  # approx ~ 1nn
knn.fit(x_train_full, y_train_full)  
%%time
  
Mconfusion = np.zeros((10, 10))  # [true, predicted]
y_predicted = []

for i in tqdm(range(len(y_test_full))):
  prediction = knn.predict([x_test_full[i]])[0]
  y_predicted.append(prediction)
  Mconfusion[y_test_full[i], prediction] += 1
_all = np.sum(np.sum(Mconfusion))
_true = np.sum([Mconfusion[i][i] for i in range(10)])

print('KNN accuracy: ', _true/_all)
report = classification_report(y_test_full, y_predicted)
print(report)
fig, ax = plt.subplots(figsize=(8,8))
sns.heatmap(Mconfusion, annot=False, ax = ax)

# By definition a confusion matrix C is such that Cij is equal to the number of observations known to be in group i but predicted to be in group j.
ax.set_xlabel('Predicted labels')
ax.set_ylabel('True labels')
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(labels)
ax.yaxis.set_ticklabels(labels)

fig.savefig('KNN_confusion_matrix.png')
plt.show()
# Manually print confusion matrix
print('  ', end='  ')
print('\033[1m'+'0    ', end='')
for j in labels[1:]:
    print(str(j), end='   ')
print('\033[0m')
for idx, i in enumerate(Mconfusion):
    print('\033[1m'+str(labels[idx])+'\033[0m', end=' ')
    for j in i:
        print('%3d' % (j), end=' ')
    print()
np.save('./KNN_Matrix', Mconfusion, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./KNN_Matrix.csv", Mconfusion, delimiter=",")<jupyter_output><empty_output><jupyter_text>Support Vector Machine (SVC)<jupyter_code>print(x_train_choose.shape)
print(y_train_choose.shape)
print(x_test_choose.shape)
print(y_test_choose.shape)
for i in range(56):  # print values of image!
  for j in range(56):
    print("%.2f " % x_train_choose[0][56*i+j], end='')
  print()<jupyter_output>1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 0.99 0.98 0.98 0.96 0.95 0.94 0.92 0.89 0.87 0.85 0.84 0.84 0.86 0.89 0.91 0.94 0.96 0.97 0.98 0.99 0.99 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 
1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 0.99 0.99 0.98 0.97 0.95 0.93 0.91 0.88 0.85 0.82 0.78 0.76 0.75 0.75 0.78 0.81 0.85 0.89 0.91 0.94 0.96 0.97 0.98 0.99 0.99 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 
1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 0.99 0.98 0.97 0.95 0.93 0.89 0.85 0.81 0.77 0.73 0.68 0.64 0.60 0.59 0.60 0.63 0.67 0.73 0.78 0.82 0.86 0.89 0.93 0.95 0.97 0.98 0.99 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 
1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 0.99 0.99 0.98 0.96 0.93 0.90 0.85 0.79 0.72 0.66 0.61 0.56 0.51 0.46 0.42 0.41 0.42 0.45 0.[...]<jupyter_text>Linearni kernel<jupyter_code>%%time
# linear SVC
model_linear = SVC(kernel='linear')
model_linear.fit(x_train_full[:24_000], y_train_full[:24_000])
# y_pred_choose = model_linear.predict(x_test_full)
y_predicted = []
for i in tqdm(range(len(x_test_full))):
  y_predicted.append(model_linear.predict([x_test_full[i]])[0])
print("accuracy:", metrics.accuracy_score(y_true=y_test_full, y_pred=y_predicted), "\n")
print(metrics.confusion_matrix(y_true=y_test_full, y_pred=y_predicted))
report = classification_report(y_test_full, y_predicted)
print(report)
fig, ax = plt.subplots(figsize=(8,8))
sns.heatmap(metrics.confusion_matrix(y_true=y_test_full, y_pred=y_predicted), annot=False, ax = ax)

# By definition a confusion matrix C is such that Cij is equal to the number of observations known to be in group i but predicted to be in group j.
ax.set_xlabel('Predicted labels')
ax.set_ylabel('True labels')
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(labels)
ax.yaxis.set_ticklabels(labels)

fig.savefig('KNN_confusion_matrix.png')
plt.show()<jupyter_output><empty_output><jupyter_text>Nelinearni kernel<jupyter_code>%%time

CS = [0.05, 0.1, 1, 5, 10, 100]
gammas = [0.01, 0.1, 0.25, 0.5, 1.0]
kernels = ['poly', 'rbf', 'sigmoid']

bestC = -1
bestGamma = -1
bestKernel = '-1'
bestScore = -1
for C in CS:
  for gamma in gammas:
    for kernel in kernels:
      model = SVC(C=C, gamma=gamma, kernel=kernel) 
      model.fit(x_train_choose, y_train_choose)
      
      score = metrics.accuracy_score(y_true=y_test_choose, y_pred=model.predict(x_test_choose))
      if score > bestScore:
        bestC = C
        bestGamma = gamma
        bestKernel = kernel
        bestScore = score
      print('C: ', C, ' gamma: ', gamma, ', kernel: ', kernel, ' = ', score)
print('bestC: ', bestC)
print('bestGamma: ', bestGamma)
print('bestKernel: ', bestKernel)
print('bestScore: ', bestScore)
%%time
model_linear = SVC(kernel=bestKernel, C=bestC, gamma=bestGamma)
model_linear.fit(x_train_full[:60_000], y_train_full[:60_000])
# y_pred_choose = model_linear.predict(x_test_full)
y_predicted = []
for i in tqdm(range(len(x_test_full))):
  y_predicted.append(model_linear.predict([x_test_full[i]])[0])
print("accuracy:", metrics.accuracy_score(y_true=y_test_full, y_pred=y_predicted), "\n")
print(metrics.confusion_matrix(y_true=y_test_full, y_pred=y_predicted))
report = classification_report(y_test_full, y_predicted)
print(report)
fig, ax = plt.subplots(figsize=(8,8))
sns.heatmap(metrics.confusion_matrix(y_true=y_test_full, y_pred=y_predicted), annot=False, ax = ax)

# By definition a confusion matrix C is such that Cij is equal to the number of observations known to be in group i but predicted to be in group j.
ax.set_xlabel('Predicted labels')
ax.set_ylabel('True labels')
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(labels)
ax.yaxis.set_ticklabels(labels)

fig.savefig('KNN_confusion_matrix.png')
plt.show()
M = metrics.confusion_matrix(y_true=y_test_full, y_pred=y_predicted)
np.save('./RBF_SVC_Matrix', M, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./RBF_SVC_Matrix.csv", M, delimiter=",")<jupyter_output><empty_output><jupyter_text>Decision Tree  Tree<jupyter_code>def visualize_tree(tree, feature_names, class_names):  # Used from <NAME>ic site
    with open("dt.dot", "w") as f:
        export_graphviz(tree, out_file=f, feature_names=feature_names, class_names=class_names, filled=True, rounded=True)
    f.close()

    subprocess.call("dot -Tpng dt.dot -o tree.png", shell=True)
#clf = tree.DecisionTreeClassifier()  # On choose datač just small check
#clf = clf.fit(x_train_choose, y_train_choose)
#y_pred_choose = clf.predict(x_test_choose)
#print(y_pred_choose[0:20], "...")
#print(y_test_choose[0:20], "...")
#print(metrics.accuracy_score(y_test_choose, y_pred_choose))
%%time

clf = tree.DecisionTreeClassifier()
clf = clf.fit(x_train_full, y_train_full)
y_predicted = clf.predict(x_test_full)
print(y_predicted[0:10], "...")
print(y_test[0:10], "...")
print(metrics.accuracy_score(y_test_full, y_predicted))
M = confusion_matrix(y_test, y_predicted)
display(M)
recall = np.diag(M) / np.sum(M, axis = 1)
print(recall)
precision = np.diag(M) / np.sum(M, axis = 0)
print(precision)
fig, ax = plt.subplots(figsize=(5,5))
sns.heatmap(M, annot=False, ax = ax)

# By definition a confusion matrix C is such that Cij is equal to the number of observations known to be in group i but predicted to be in group j.
ax.set_xlabel('Predicted labels')
ax.set_ylabel('True labels')
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(labels)
ax.yaxis.set_ticklabels(labels)

fig.savefig('confusion_matrix.png')
plt.show()
# Manually print confusion matrix
print('  ', end='  ')
print('\033[1m'+'0    ', end='')
for j in labels[1:]:
    print(str(j), end='   ')
print('\033[0m')
for idx, i in enumerate(M):
    print('\033[1m'+str(labels[idx])+'\033[0m', end=' ')
    for j in i:
        print('%3d' % (j), end=' ')
    print()
print('predicted: ', y_predicted[0:5])
print('test: ', y_test[0:5])
acc = accuracy_score(y_test, y_predicted)
print(acc*100.0)
report = classification_report(y_test, y_predicted)
print(report)
np.save('./DT_Matrix', M, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./DT_Matrix.csv", M, delimiter=",")<jupyter_output><empty_output><jupyter_text>Pixel Importance Visualisation<jupyter_code>visualize_tree(clf,['pixel '+str((i,j)) for i in range(56) for j in range(56)], ['0','1','2','3','4','5','6','7','8','9'])
len([(i,j) for i in range(56) for j in range(56)])
print('feature (pixel) importances:', '\n')  # Interpretability of decision tree
pd.Series(clf.feature_importances_, index=['pixel '+str((i,j)) for i in range(56) for j in range(56)]).sort_values(ascending=False)
# Plot it
importance = pd.Series(clf.feature_importances_, index=['pixel '+str((i,j)) for i in range(56) for j in range(56)])
plt.figure(figsize=(7,7))
image_importance = (importance.values).reshape((56, 56))
plt.imshow(image_importance, cmap='gray')
plt.show()<jupyter_output><empty_output><jupyter_text>Random forests  100 + gini<jupyter_code>%%time
clf = RandomForestClassifier(n_estimators=100, criterion='gini')
clf.fit(x_train_full, y_train_full)
acc = accuracy_score(y_test_full, y_predicted)
print('accuracy: ', acc*100.0)
report = classification_report(y_test_full, y_predicted)
print(report)

M = confusion_matrix(y_test_full, y_predicted)
# Manually print confusion matrix
print('  ', end='  ')
print('\033[1m'+'0    ', end='')
for j in labels[1:]:
    print(str(j), end='   ')
print('\033[0m')
for idx, i in enumerate(Mconfusion):
    print('\033[1m'+str(labels[idx])+'\033[0m', end=' ')
    for j in i:
        print('%3d' % (j), end=' ')
    print()
np.save('./RF_100gini_Matrix', M, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./RF_100gini_Matrix.csv", M, delimiter=",")<jupyter_output><empty_output><jupyter_text>100 + entropy<jupyter_code>%%time
clf = RandomForestClassifier(n_estimators=100, criterion='entropy')
clf.fit(x_train_full, y_train_full)
acc = accuracy_score(y_test_full, y_predicted)
print('accuracy: ', acc*100.0)
report = classification_report(y_test_full, y_predicted)
print(report)

M = confusion_matrix(y_test_full, y_predicted)
# Manually print confusion matrix
print('  ', end='  ')
print('\033[1m'+'0    ', end='')
for j in labels[1:]:
    print(str(j), end='   ')
print('\033[0m')
for idx, i in enumerate(Mconfusion):
    print('\033[1m'+str(labels[idx])+'\033[0m', end=' ')
    for j in i:
        print('%3d' % (j), end=' ')
    print()
np.save('./RF_100entropy_Matrix', M, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./RF_100entropy_Matrix.csv", M, delimiter=",")<jupyter_output><empty_output><jupyter_text>Deep Neural Network  Architecture<jupyter_code>BOX_SIZE = 56                        
image_shape = (BOX_SIZE, BOX_SIZE, 1)  
labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
number_of_classes = len(labels)        
x_input = Input(shape=image_shape)  

x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x_input)
x = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x) 
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=(2,2), padding='valid')(x)                                                      
x = Conv2D(filters=48, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x)        
x = Conv2D(filters=48, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x) 
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=(2,2), padding='valid')(x)                                                      
x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x)        
x = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), activation='relu', padding='valid')(x) 
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=(3, 3), padding='valid')(x)                                                    
x = Flatten()(x)                                                                                       

x = Dense(units=768, activation='relu')(x)         
x = Dropout(rate=0.25)(x)                          
x = Dense(units=256, name='next_to_last')(x)                           
x = Dropout(rate=0.25)(x)                          

y_output = Dense(units=number_of_classes, activation='softmax')(x)  
recognizer = Model(inputs=x_input, outputs=y_output, name='recognizer')
recognizer.summary()
recognizer.compile(optimizer=Adam(lr=1e-3), loss=categorical_crossentropy, metrics=['accuracy'])   # beta_1=0.9, beta_2=0.999
reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, verbose=1, mode='min', #add min_delta
                           cooldown=1, min_le=1e-8)

estop = EarlyStopping(monitor='val_loss', min_delta=0, patience=4, verbose=1, mode='min')

excel = CSVLogger(filename='./CNN.csv', separator=',', append=False)

checkp = ModelCheckpoint('./CNN_weights.h5', monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='auto', period=1)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>class SequenceGenerator(Sequence):
  # infodisk - [(path, label, writer)]
  # batch_size - size of batch
  
    def __init__(self, xs, ys, batch_size):
      self.xs = np.array(xs)
      self.ys = np.array(ys)
      self.batch_size = batch_size
    
    def __len__(self):
      return int(np.ceil(len(self.xs)/float(self.batch_size)))
  
    def __getitem__(self, index):
      _xs = self.xs[index*self.batch_size: (index+1)*self.batch_size]
      _ys = self.ys[index*self.batch_size: (index+1)*self.batch_size]
    
      return np.array([(x.reshape(BOX_SIZE, BOX_SIZE, 1))/255.0 for x in _xs]), to_categorical(_ys, number_of_classes)
  
    def on_epoch_end(self):
      self.xs, self.ys = shuffle(self.xs, self.ys)
test = SequenceGenerator(x_train, y_train, 50)
x, y = test.__getitem__(0)
print(x.shape)
print(y.shape)
np.array(x_train).shape
train_generator = SequenceGenerator(x_train[:100_000], y_train[:100_000], 1000)
validation_generator = SequenceGenerator(x_train[100_000:], y_train[100_000:], 1000)
history = recognizer.fit_generator(generator=train_generator, epochs=15, verbose=1,
                                   use_multiprocessing=False, workers=2, validation_data=validation_generator,
                                   callbacks=[reduce, estop, excel, checkp])
recognizer.evaluate(x=(np.array(x_test)/255.0).reshape(len(x_test), BOX_SIZE, BOX_SIZE, 1), y=to_categorical(np.array(y_test), number_of_classes), batch_size=256, verbose=1)
recognizer.save_weights('./CNN_weights.h5') 
recognizer.save('./CNN_model.h5')<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>epochs = history.epoch
loss = history.history['loss']
val_loss = history.history['val_loss']

plt.title('Loss/Val loss curve')
plt.xlabel('epochs')
plt.ylabel('loss')
plt.plot(epochs, loss, color='red', label='training')
plt.plot(epochs, val_loss, color='orange', label='validation')
plt.legend()
plt.show()
acc = history.history['acc']
val_acc = history.history['val_acc']

plt.title('Acc/Val acc curve')
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.plot(epochs, acc, color='red', label='training')
plt.plot(epochs, val_acc, color='orange', label='validation')
plt.legend()
plt.show()
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report

y_predicted = np.argmax(recognizer.predict((np.array(x_test)/255.0).reshape(len(x_test), BOX_SIZE, BOX_SIZE, 1)), axis=1)
print('predicted: ', y_predicted[0:5])
print('test: ', y_test[0:5])
acc = accuracy_score(y_test, y_predicted)
print(acc*100.0)
report = classification_report(y_test, y_predicted)
print(report)
from sklearn.metrics import confusion_matrix

M = confusion_matrix(y_test, y_predicted)
display(M)
recall = np.diag(M) / np.sum(M, axis = 1)
print(recall)
precision = np.diag(M) / np.sum(M, axis = 0)
print(precision)
import seaborn as sns

fig, ax = plt.subplots(figsize=(5,5))
sns.heatmap(M, annot=False, ax = ax)

# By definition a confusion matrix C is such that Cij is equal to the number of observations known to be in group i but predicted to be in group j.
ax.set_xlabel('Predicted labels')
ax.set_ylabel('True labels')
ax.set_title('Confusion Matrix');
ax.xaxis.set_ticklabels(labels)
ax.yaxis.set_ticklabels(labels)

fig.savefig('confusion_matrix.png')
plt.show()
# Manually print confusion matrix
print('  ', end='  ')
print('0    ', end='')
for j in labels[1:]:
    print(str(j), end='   ')
print()
for idx, i in enumerate(M):
    print(str(labels[idx]), end=' ')
    for j in i:
        print('%3d' % (j), end=' ')
    print()
# Visualise missclassified examples 
indexes = y_test!=y_predicted
n = sum(indexes)  # 126
x_test_wrong = np.array(x_test)[indexes]
y_test_wrong = np.array(y_test)[indexes]
y_predicted_wrong = np.array(y_predicted)[indexes]
for i in range(10):
  print('true label: ', y_test_wrong[i])
  print('predicted label: ', y_predicted_wrong[i])
  plt.imshow(x_test_wrong[i], cmap='gray')
  plt.show()
np.save('./CNN_Matrix', M, allow_pickle=False)  # Save confusion matrix .npz and .csv 
np.savetxt("./CNN_Matrix.csv", M, delimiter=",")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15354700854700853, 'fraction_numerical': 0.04987179487179487, 'mean_word_length': 4.74539651362632, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '325115', 'license': 'MIT', 'n_tokens_mistral': 9651, 'n_tokens_neox': 8831, 'n_words': 1940, 'path': 'src/DoubledMNIST_Classification_Tasks.ipynb', 'repo_name': 'MilanCugur/DoubledMNIST', 'size': '23409'}