<jupyter_start><jupyter_text>Fashion-MNIST is a dataset of Zalando's article images—consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes.  It shares the same image size and structure of training and testing splits. -  Try to build a classifier for the Fashion-MNIST dataset that achieves over 85% accuracy on the test set. -  Use only classifiers that are used in the Chapter 3 of the textbook.-  Do the error analysis following the textbook.<jupyter_code># Check GPU
import tensorflow as tf
tf.test.gpu_device_name()
# !mkdir -p my_drive
# !google-drive-ocamlfuse my_drive
# #!mkdir -p my_drive/Fashion
# !python3 /content/my_drive/Fashion/mnist_reader.py
import sys
sys.path.append('/content/my_drive/Fashion')<jupyter_output><empty_output><jupyter_text>----------Everything above this is for Google Coloab----------<jupyter_code>import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
import mnist_reader
X_train, y_train = mnist_reader.load_mnist('/content/my_drive/Fashion', kind='train')
X_test, y_test = mnist_reader.load_mnist('/content/my_drive/Fashion', kind='t10k')
X_train.shape
y_train.shape<jupyter_output><empty_output><jupyter_text>LabelsEach training and test example is assigned to one of the following labels:Label	Description- 0	T-shirt/top- 1	Trouser- 2	Pullover- 3	Dress- 4	Coat- 5	Sandal- 6	Shirt- 7	Sneaker- 8	Bag- 9	Ankle boot<jupyter_code>def plot_digit(data):
    image = data.reshape(28, 28)
    plt.imshow(image, cmap = matplotlib.cm.binary, interpolation="nearest")
    plt.axis("off")
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt

some_digit = X_train[36001]
some_digit_image = some_digit.reshape(28, 28)
plt.imshow(some_digit_image, cmap = matplotlib.cm.binary, interpolation="nearest")
plt.axis("off");
y_train[36001]
plot_digit(X_train[40000])
y_train[40000]
def plot_digits(instances, images_per_row=10, **options):
    size = 28
    images_per_row = min(len(instances), images_per_row)
    images = [instance.reshape(size,size) for instance in instances]
    n_rows = (len(instances) - 1) // images_per_row + 1
    row_images = []
    n_empty = n_rows * images_per_row - len(instances)
    images.append(np.zeros((size, size * n_empty)))
    for row in range(n_rows):
        rimages = images[row * images_per_row : (row + 1) * images_per_row]
        row_images.append(np.concatenate(rimages, axis=1))
    image = np.concatenate(row_images, axis=0)
    plt.imshow(image, cmap = matplotlib.cm.binary, **options)
    plt.axis("off")
plt.figure(figsize=(9,9))
X_0 = X_train[(y_train == 0)]
example_images = X_0[:100]
plot_digits(example_images, images_per_row=10)
plt.figure(figsize=(9,9))
X_6 = X_train[(y_train == 6)]
example_images = X_6[:100]
plot_digits(example_images, images_per_row=10)
plt.figure(figsize=(9,9))
X_1 = X_train[(y_train == 1)]
example_images = X_1[:100]
plot_digits(example_images, images_per_row=10)
plt.figure(figsize=(9,9))
X_2 = X_train[(y_train == 2)]
example_images = X_2[:100]
plot_digits(example_images, images_per_row=10)
plt.figure(figsize=(9,9))
X_3 = X_train[(y_train == 3)]
example_images = X_3[:100]
plot_digits(example_images, images_per_row=10)
some_article = X_train[1014]
plot_digit(some_article)
y_train[1014]<jupyter_output><empty_output><jupyter_text>Training a Binary classifier to identify the shirt<jupyter_code>y_train_shirt = (y_train == 6)
y_test_shirt = (y_train == 6)
from sklearn.linear_model import SGDClassifier

sgd_clf = SGDClassifier(max_iter=1000, tol=1e-3, random_state=42)
sgd_clf.fit(X_train, y_train_shirt)<jupyter_output><empty_output><jupyter_text>Performance measures<jupyter_code>from sklearn.model_selection import cross_val_score

cross_val_score(sgd_clf, X_train, y_train_shirt, cv=3, scoring="accuracy")
# from sklearn.model_selection import StratifiedKFold
# from sklearn.base import clone

# skfolds = StratifiedKFold(n_splits=3, random_state=42)

# for train_index, test_index in skfolds.split(X_train, y_train_sneaker):
#     clone_clf = clone(sgd_clf)
#     X_train_folds = X_train[train_index]
#     y_train_folds = y_train_sneaker[train_index]
#     X_test_fold = X_train[test_index]
#     y_test_fold = y_train_sneaker[test_index]

#     clone_clf.fit(X_train_folds, y_train_folds)
#     y_pred = clone_clf.predict(X_test_fold)
#     n_correct = sum(y_pred == y_test_fold)
#     print(n_correct / len(y_pred))
from sklearn.model_selection import cross_val_predict

y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_shirt, cv=3)
from sklearn.metrics import confusion_matrix

confusion_matrix(y_train_shirt, y_train_pred)
y_train_perfect_predictions = y_train_shirt  # pretend we reached perfection
confusion_matrix(y_train_shirt, y_train_perfect_predictions)
from sklearn.metrics import precision_score, recall_score

precision_score(y_train_shirt, y_train_pred)
recall_score(y_train_shirt, y_train_pred)
from sklearn.metrics import f1_score

f1_score(y_train_shirt, y_train_pred)
y_scores = cross_val_predict(sgd_clf, X_train, y_train_shirt, cv=3, method="decision_function")
from sklearn.metrics import precision_recall_curve

precisions, recalls, thresholds = precision_recall_curve(y_train_shirt, y_scores)
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision", linewidth=2)
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall", linewidth=2)
    plt.legend(loc="center right", fontsize=16) # Not shown in the book
    plt.xlabel("Threshold", fontsize=16)        # Not shown
    plt.grid(True)                              # Not shown
    plt.axis([-50000, 50000, 0, 1])             # Not shown

plt.figure(figsize=(8, 4))                      # Not shown
plot_precision_recall_vs_threshold(precisions, recalls, thresholds)

#save_fig("precision_recall_vs_threshold_plot")  # Not shown
plt.show()
def plot_precision_vs_recall(precisions, recalls):
    plt.plot(recalls, precisions, "b-", linewidth=2)
    plt.xlabel("Recall", fontsize=16)
    plt.ylabel("Precision", fontsize=16)
    plt.axis([0, 1, 0, 1])
    plt.grid(True)

plt.figure(figsize=(8, 6))
plot_precision_vs_recall(precisions, recalls)

#save_fig("precision_vs_recall_plot")
plt.show()<jupyter_output><empty_output><jupyter_text>Sneaker Binary Classifier for comparison<jupyter_code>y_train_sneaker = (y_train == 7)
y_scores_sneaker = cross_val_predict(sgd_clf, X_train, y_train_sneaker, cv=3, method="decision_function")
precisions, recalls, thresholds = precision_recall_curve(y_train_sneaker, y_scores_sneaker)
plt.figure(figsize=(8, 4))                      # Not shown
plot_precision_recall_vs_threshold(precisions, recalls, thresholds)
#save_fig("precision_recall_vs_threshold_plot")  # Not shown
plt.show()
plt.figure(figsize=(8, 6))
plot_precision_vs_recall(precisions, recalls)

plt.show()<jupyter_output><empty_output><jupyter_text>ROC Curves<jupyter_code>from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_train_shirt, y_scores)
def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--') # dashed diagonal
    plt.axis([0, 1, 0, 1])                                    # Not shown in the book
    plt.xlabel('False Positive Rate (Fall-Out)', fontsize=16) # Not shown
    plt.ylabel('True Positive Rate (Recall)', fontsize=16)    # Not shown
    plt.grid(True)                                            # Not shown

plt.figure(figsize=(8, 6))                         # Not shown
plot_roc_curve(fpr, tpr)

#save_fig("roc_curve_plot")                         # Not shown
plt.show()
from sklearn.metrics import roc_auc_score

roc_auc_score(y_train_shirt, y_scores)
from sklearn.ensemble import RandomForestClassifier
forest_clf = RandomForestClassifier(n_estimators=100, random_state=42)
y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_shirt, cv=3, method="predict_proba")
y_scores_forest = y_probas_forest[:, 1] # score = proba of positive class
fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_shirt,y_scores_forest)
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, "b:", linewidth=2, label="SGD")
plot_roc_curve(fpr_forest, tpr_forest, "Random Forest")

plt.grid(True)
plt.legend(loc="lower right", fontsize=16)
#save_fig("roc_curve_comparison_plot")
plt.show()
roc_auc_score(y_train_shirt, y_scores_forest)
y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_shirt, cv=3)
precision_score(y_train_shirt, y_train_pred_forest)
recall_score(y_train_shirt, y_train_pred_forest)<jupyter_output><empty_output><jupyter_text>Multiclass classification<jupyter_code>#from sklearn.svm import SVC
#from sklearn.multiclass import OneVsRestClassifier
#ovr_clf = OneVsRestClassifier(SVC(gamma="auto", random_state=42))
#ovr_clf.fit(X_train, y_train)
forest_clf = RandomForestClassifier(n_estimators=100, random_state=42)
forest_clf.fit(X_train, y_train)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))
y_train_pred_forest = cross_val_predict(forest_clf, X_train_scaled, y_train, cv=3)
from sklearn.metrics import accuracy_score
accuracy_score(y_train, y_train_pred_forest)<jupyter_output><empty_output><jupyter_text>88% Accuracy<jupyter_code>#cross_val_score(forest_clf, X_train_scaled, y_train, cv=3, scoring="accuracy")<jupyter_output><empty_output><jupyter_text>Error Analysis<jupyter_code>conf_mx = confusion_matrix(y_train, y_train_pred_forest)
conf_mx
plt.matshow(conf_mx, cmap=plt.cm.gray)
plt.show()
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)
plt.matshow(norm_conf_mx, cmap=plt.cm.gray)
plt.show()<jupyter_output><empty_output><jupyter_text>6(shirt) is confused with 0(T-shirt/top) the most<jupyter_code>cl_a, cl_b = 6, 0
X_aa = X_train[(y_train == cl_a) & (y_train_pred_forest == cl_a)]
X_ab = X_train[(y_train == cl_a) & (y_train_pred_forest == cl_b)]
X_ba = X_train[(y_train == cl_b) & (y_train_pred_forest == cl_a)]
X_bb = X_train[(y_train == cl_b) & (y_train_pred_forest == cl_b)]

plt.figure(figsize=(8,8))
plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)
plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)
plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)
plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)
plt.show()<jupyter_output><empty_output><jupyter_text>The error / loss in accuraccy is mostly coming from the shirt being confused for a T-shirt or top. This confusion is from the length of the sleeves and is not easy to tell.  Testing test sample<jupyter_code>y_test_pred = forest_clf.predict(X_test)
accuracy_score(y_test, y_test_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1143991338866835, 'fraction_numerical': 0.023908336340671237, 'mean_word_length': 6.083067092651757, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '4283', 'license': 'MIT', 'n_tokens_mistral': 4409, 'n_tokens_neox': 4085, 'n_words': 794, 'path': 'Fashion MNIST/Linear Model/Fashion_MNIST_oneill_complete.ipynb', 'repo_name': 'aj96oneill/ML-Beginner-Projects', 'size': '11084'}