<jupyter_start><jupyter_text>Training a Binary Classifier<jupyter_code>y_train_5 = (y_train == 5) # True for all 5s, False for all other digits
y_test_5 = (y_test == 5)
from sklearn.linear_model import SGDClassifier

sgd_clf = SGDClassifier(random_state=42)
sgd_clf.fit(X_train, y_train_5)
len(X_train) == len(y_train_5)
sgd_clf.predict([some_digit])<jupyter_output><empty_output><jupyter_text>Performance Measures  Measuring Accuracy Using Cross-Validation Implementing Cross-Validation<jupyter_code>from sklearn.model_selection import StratifiedKFold
from sklearn.base import clone

skfolds = StratifiedKFold(n_splits=3, random_state=42)

for train_index, test_index in skfolds.split(X_train, y_train_5):
    clone_clf = clone(sgd_clf)
    X_train_folds = X_train[train_index]
    y_train_folds = y_train_5[train_index]
    X_test_fold = X_train[test_index]
    y_test_fold = y_train_5[test_index]
    
    clone_clf.fit(X_train_folds, y_train_folds)
    y_pred = clone_clf.predict(X_test_fold)
    n_correct = sum(y_pred == y_test_fold)
    print(n_correct / len(y_pred)) # prints 0.9502, 0.9656 and 0.96495
from sklearn.model_selection import cross_val_score

cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring='accuracy')
from sklearn.base import BaseEstimator

class Never5Classifier(BaseEstimator):
    def fit(self, X, y='None'):
        return self
    def predict(self, X):
        return np.zeros((len(X), 1), dtype=bool)
never_5_clf = Never5Classifier()
cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring='accuracy')<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code># page 90
from sklearn.model_selection import cross_val_predict

y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)
from sklearn.metrics import confusion_matrix

confusion_matrix(y_train_5, y_train_pred)
y_train_perfect_predictions = y_train_5 # pretend we reached perfection
confusion_matrix(y_train_5, y_train_perfect_predictions)<jupyter_output><empty_output><jupyter_text>Precision and Recall<jupyter_code>from sklearn.metrics import precision_score, recall_score

precision_score(y_train_5, y_train_pred) # 4096 / (4096 + 1522)
recall_score(y_train_5, y_train_pred)
from sklearn.metrics import f1_score

f1_score(y_train_5, y_train_pred)<jupyter_output><empty_output><jupyter_text>Precision/Recall Trade-off<jupyter_code>y_scores = sgd_clf.decision_function([some_digit])
y_scores
threshold = 0
y_some_digit_pred = (y_scores > threshold)
y_some_digit_pred
threshold = 8000
y_some_digit_pred = (y_scores > threshold)
y_some_digit_pred
y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3,
                             method='decision_function')
from sklearn.metrics import precision_recall_curve

precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)
def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):
    plt.plot(thresholds, precisions[:-1], "b--", label='Precision')
    plt.plot(thresholds, recalls[:-1], "g-", label='Recall')
    # highlight the threshold and add the legend, axis label and grid

plot_precision_recall_vs_threshold(precisions, recalls, thresholds)
plt.show()
threshold_90_precision = thresholds[np.argmax(precisions >= 0.90)] # ~7816
threshold_90_precision #????
y_train_pred_90 = (y_scores >= threshold_90_precision)
precision_score(y_train_5, y_train_pred_90)
recall_score(y_train_5, y_train_pred_90)<jupyter_output><empty_output><jupyter_text>A high-precision classifier is not very useful if its recall is too low!  The ROC Curve<jupyter_code>from sklearn.metrics import roc_curve

fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)
def plot_roc_curve(fpr, tpr, label=None):
    plt.plot(fpr, tpr, linewidth=2, label=label)
    plt.plot([0,1], [0,1], 'k--') # Dashed diagonal
    #Add axis labels and grid
plot_roc_curve(fpr, tpr)
plt.show()
from sklearn.metrics import roc_auc_score

roc_auc_score(y_train_5, y_scores)<jupyter_output><empty_output><jupyter_text>Let's train a RandomForestClassifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

forest_clf = RandomForestClassifier(random_state=42)
y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,
                                    method='predict_proba')
y_scores_forest = y_probas_forest[:,1] # score = proba of positive class
fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5, y_scores_forest)
plt.plot(fpr, tpr, "b:", label='SGD')
plot_roc_curve(fpr_forest, tpr_forest, "Random Forest")
plt.legend(loc='lower right')
plt.show()
roc_auc_score(y_train_5, y_scores_forest)
precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores_forest)
# threshold_90_precision = thresholds[np.argmax(precisions >= 0.90)] 
# threshold_90_precision #????
# y_train_pred_90 = (y_scores >= threshold_90_precision)<jupyter_output><empty_output><jupyter_text>this is should have a precision of 99.0% and a recall of 86.6% find out what happened here<jupyter_code># precision_score(y_train_5, y_train_pred_90)
# recall_score(y_train_5, y_train_pred_90)<jupyter_output><empty_output><jupyter_text>Multiclass Classification<jupyter_code>from sklearn.svm import SVC

svm_clf = SVC()
svm_clf.fit(X_train, y_train) # y_train, not y_train_5
svm_clf.predict(([some_digit]))<jupyter_output><empty_output><jupyter_text>see pg 101 for details:add notes<jupyter_code>some_digit_scores = svm_clf.decision_function([some_digit])
some_digit_scores
np.argmax(some_digit_scores)
svm_clf.classes_
svm_clf.classes_[5]<jupyter_output><empty_output><jupyter_text>OvR<jupyter_code>### this part of notebook takes some time to run
from sklearn.multiclass import OneVsRestClassifier

ovr_clf = OneVsRestClassifier(SVC())
ovr_clf.fit(X_train, y_train)
ovr_clf.predict([some_digit])
len(ovr_clf.estimators_)<jupyter_output><empty_output><jupyter_text>SGD classifier<jupyter_code>sgd_clf.fit(X_train, y_train)
sgd_clf.predict([some_digit])<jupyter_output><empty_output><jupyter_text>This time Scikit-Learn did not have to tun OvR or OvO because SGD classifiers can directly classify instances into multiple classes<jupyter_code>sgd_clf.decision_function([some_digit])
cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring='accuracy')
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))
cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring='accuracy')<jupyter_output><empty_output><jupyter_text>Error Analysis<jupyter_code>y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)
conf_mx = confusion_matrix(y_train, y_train_pred)
conf_mx
plt.matshow(conf_mx, cmap=plt.cm.gray)
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)
plt.matshow(norm_conf_mx, cmap=plt.cm.gray)
plt.show()
cl_a, cl_b = 3, 5
X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]
X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]
X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]
X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]
plt.figure(figsize=(8,8))
plt.subplot(221); plot_digits(X_aa[:25], images_per_row=5)
plt.subplot(222); plot_digits(X_ab[:25], images_per_row=5)
plt.subplot(223); plot_digits(X_ba[:25], images_per_row=5)
plt.subplot(224); plot_digits(X_bb[:25], images_per_row=5)
plt.show()<jupyter_output><empty_output><jupyter_text>Multilabel Classification<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

y_train_large = (y_train >= 7)
y_train_odd = (y_train % 2 == 1)
y_multilabel = np.c_[y_train_large, y_train_odd]

knn_clf = KNeighborsClassifier()
knn_clf.fit(X_train, y_multilabel)
knn_clf.predict([some_digit])
y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)
f1_score(y_multilabel, y_train_knn_pred, average-"marco")<jupyter_output><empty_output><jupyter_text>Multiouput Classification<jupyter_code>noise = np.random.randint(0, 100, (len(X_train), 784))
X_train_mod = X_train + noise
noise = np.random.randint(0, 100, (len(X_test), 784))
X_test_mod = X_test + noise
y_train_mode = X_train
y_test_mode = X_test
knn_clf.fit(X_train_mod, y_train_mod)
clean_digit = knn_clf.predict([X_test_mod[some_index]])
plot_digit(clean_digit)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1072801537722249, 'fraction_numerical': 0.023065833733781835, 'mean_word_length': 8.068627450980392, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '66640', 'license': 'MIT', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3240, 'n_words': 541, 'path': 'MNIST_classification.ipynb', 'repo_name': 'JeffreyAsuncion/MachineLearning_OREILLY', 'size': '8324'}