<jupyter_start><jupyter_text>Visualize image<jupyter_code>random_image = x[0]
random_image_2d = random_image.reshape(28, 28)
plt.imshow(random_image_2d, cmap=mpl.cm.binary)
plt.axis("off")
plt.show()
plt.imshow(x[123].reshape(28,28), cmap=mpl.cm.binary)
x_train, x_test, y_train, y_test = x[:60000], x[60000:], y[:60000], y[60000:]<jupyter_output><empty_output><jupyter_text>One-digit detector (binary detector)<jupyter_code>y_train_5 = (y_train == 5) # True for all 5s, False for all other digits.
y_test_5 = (y_test == 5)
model = SGDClassifier(random_state=42)
model.fit(x_train, y_train_5)
model.predict([x_train[0]])<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>cross_val_score(model, x_train, y_train_5, cv = 3, scoring='accuracy')<jupyter_output><empty_output><jupyter_text>This is pretty high accuracy, but we cannot be below 0.9 because we have only 10% images with five digit.  Confusion Matrix<jupyter_code>predicts = cross_val_predict(model, x_train, y_train_5, cv=3) # Clean prediction - the prediction is made by a model that never saw the data during training.
predicts
confusion_matrix(y_train_5, predicts) ## [TRUE Negative, False Positive],[False Negative, True Positive]
confusion_matrix(y_train_5, y_train_5) # Perfect predictions<jupyter_output><empty_output><jupyter_text>Precision/Recall Tradeoff<jupyter_code>precision_score(y_train_5, predicts), recall_score(y_train_5, predicts), f1_score(y_train_5, predicts)
predict_scores = cross_val_predict(model, x_train, y_train_5, cv=3, method='decision_function')
predict_scores
precisions, recalls, thresholds = precision_recall_curve(y_train_5, predict_scores)
precisions, recalls, thresholds
threshold_90_precision = thresholds[np.argmax(precisions >= 0.80)]
recall_90_precision = recalls[np.argmax(precisions >= 0.80)]

y_train_pred_90 = (predict_scores >= threshold_90_precision)
def plot_precision_recall_vs_threshold_by_precision(precisions, recalls, thresholds, precision):
    threshold_90_precision = thresholds[np.argmax(precisions >= precision)]
    recall_90_precision = recalls[np.argmax(precisions >= precision)]
    plt.plot(thresholds, precisions[:-1], "b--", label="Precision")
    plt.plot(thresholds, recalls[:-1], "g-", label="Recall")
    plt.legend(loc="center right", fontsize=16)
    plt.xlabel("Threshold", fontsize=16)
    plt.grid(True)
    plt.axis([-50000, 50000, 0, 1])
    plt.plot([threshold_90_precision, threshold_90_precision], [0., precision], "r:")
    plt.plot([-50000, threshold_90_precision], [precision, precision], "r:")
    plt.plot([-50000, threshold_90_precision], [recall_90_precision, recall_90_precision], "r:")
    plt.plot([threshold_90_precision], [precision], "ro")
    plt.plot([threshold_90_precision], [recall_90_precision], "ro")
    plt.show()
    
plot_precision_recall_vs_threshold_by_precision(precisions, recalls, thresholds, 0.8370879772350012)
precision_score(y_train_5, y_train_pred_90), recall_score(y_train_5, y_train_pred_90)
#If someone says “let’s reach 99% precision,” you should ask, “atwhat recall?”<jupyter_output><empty_output><jupyter_text>Changing threshold<jupyter_code>threshold_at_precision = thresholds[np.argmax(precisions >= 0.95)] # Find threshold for specific precision
threshold_at_precision
predicts_at_precision = (predict_scores >= threshold_at_precision)
predicts_at_precision
precision_score(y_train_5, predicts_at_precision), recall_score(y_train_5, predicts_at_precision), f1_score(y_train_5, predicts_at_precision)<jupyter_output><empty_output><jupyter_text>ROC Curve<jupyter_code>fpr, tpr, thresholds = roc_curve(y_train_5, predict_scores)
fpr, tpr, thresholds
def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0, 1], [0, 1], 'k--') # dashed diagonal
plot_roc_curve(fpr, tpr)
roc_auc_score(y_train_5, predict_scores)
forest_clf = RandomForestClassifier(random_state=42)
forest_probas = cross_val_predict(forest_clf, x_train, y_train_5, cv=3, method='predict_proba')
forest_scores = forest_probas[:, 1]
fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, forest_scores)
plt.plot(fpr, tpr, "b:", label="SGD")
plot_roc_curve(fpr_forest, tpr_forest, "Random Forest")
plt.legend(loc="lower right")
plt.show()
roc_auc_score(y_train_5, forest_scores)
forest_predicts = cross_val_predict(forest_clf, x_train, y_train_5, cv=3)
precision_score(y_train_5, forest_predicts), recall_score(y_train_5, forest_predicts)<jupyter_output><empty_output><jupyter_text>Multiclass Classification Strategies: **OneVsOneClassifier OneVsRestClassifier**. Support Vector Machine classifiers or Linear classifiers) are strictly binary classifiers. Random Forest classifiers or naive Bayes classifiers are capable of handling multiple classes directly.  TODO: Compare SGD Classifier with Random Forest Clasifier<jupyter_code>m_sgd_classifier = SGDClassifier(random_state=42)
cross_val_score(m_sgd_classifier, x_train, y_train, cv = 3, scoring='accuracy')
m_forest_classifier = RandomForestClassifier()
cross_val_score(m_forest_classifier, x_train, y_train, cv=3, scoring='accuracy')
m_forest_predicts = cross_val_predict(m_forest_classifier, x_train, y_train, cv=3)
conf_mx = confusion_matrix(y_train, m_forest_predicts)
conf_mx
plt.matshow(conf_mx, cmap=plt.cm.gray)
plt.show()
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)
plt.matshow(norm_conf_mx, cmap=plt.cm.gray)
plt.show()<jupyter_output><empty_output><jupyter_text>Multilabel Classification<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
y_train_large = (y_train >= 7)
y_train_odd = (y_train % 2 == 1)
y_multilabel = np.c_[y_train_large, y_train_odd]
knn_clf = KNeighborsClassifier()
knn_clf.fit(x_train, y_multilabel)
index = 7
y_train[index], knn_clf.predict([x_train[index]])
knn_predicts = cross_val_predict(knn_clf, x_train, y_multilabel, cv=3)
f1_score(y_multilabel, knn_predicts, average="macro")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11651941990331723, 'fraction_numerical': 0.03150525087514586, 'mean_word_length': 9.118043844856661, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '531629', 'license': 'MIT', 'n_tokens_mistral': 2397, 'n_tokens_neox': 2160, 'n_words': 392, 'path': 'ml/classification_mnist.ipynb', 'repo_name': 'VladiGGeorgiev/ml-playground', 'size': '5999'}