<jupyter_start><jupyter_text>Exercise 9.<jupyter_code>from sklearn.datasets import fetch_openml

mnist = fetch_openml('mnist_784', version=1)
mnist.target = mnist.target.astype(np.uint8)
X_train = mnist['data'][:60000]
y_train = mnist['target'][:60000]

X_test = mnist['data'][60000:]
y_test = mnist['target'][60000:]
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.model_selection import GridSearchCV
def time_a_model(model, trainset, testset=None, predict=False):
    X_train, y_train = trainset
    t1 = time.time()
    model.fit(X_train, y_train)
    t2 = time.time()
    print(f'Fitting {model.__class__.__name__} with sample shape {X_train.shape} took {t2-t1:.1f} seconds.')
    if predict:
        X_test, y_test = testset
        y_predicted = model.predict(X_test)
        print(f'Accuracy on test set: {accuracy_score(y_test, y_predicted):.3f}')
        return y_predicted<jupyter_output><empty_output><jupyter_text>Train a random forest directly<jupyter_code>rnd_clf = RandomForestClassifier(n_estimators=200, min_samples_split=5, random_state=42, n_jobs=2)

y_predicted = time_a_model(rnd_clf, (X_train, y_train), (X_test, y_test), True)<jupyter_output>Fitting RandomForestClassifier with sample shape (60000, 784) took 37.7 seconds.
Accuracy on test set: 0.970<jupyter_text>Train PCA and then random forest<jupyter_code>pipeline = Pipeline([
    ('pca', PCA(n_components=0.95)),
    ('rnd', rnd_clf)
])

y_predicted_pca = time_a_model(pipeline, (X_train, y_train), (X_test, y_test), True)<jupyter_output>Fitting Pipeline with sample shape (60000, 784) took 98.1 seconds.
Accuracy on test set: 0.951<jupyter_text>Train a logistic regression with softmax<jupyter_code>from sklearn.linear_model import LogisticRegression

log_reg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=42)

_ = time_a_model(log_reg, (X_train, y_train), (X_test, y_test), True)<jupyter_output>Fitting LogisticRegression with sample shape (60000, 784) took 18.6 seconds.
Accuracy on test set: 0.925<jupyter_text>Train PCA and then a logistic regression with softmax<jupyter_code>t1 = time.time()
pca = PCA(n_components=0.95)
X_reduced = pca.fit_transform(X_train)
X_test_reduced = pca.transform(X_test)
t2 = time.time()
print(f"{t2-t1:.1f} seconds for using PCA in transforming trainset")
_ = time_a_model(log_reg, (X_reduced, y_train), (X_test_reduced, y_test), True)<jupyter_output>5.5 seconds for using PCA in transforming trainset
Fitting LogisticRegression with sample shape (60000, 154) took 5.6 seconds.
Accuracy on test set: 0.920<jupyter_text>Exercise 10. t-SNE and umap  t-SNE<jupyter_code>sample_indices = np.random.choice(X_train.shape[0], size=6000, replace=False)
train_idx = sample_indices[:5000]
test_idx = sample_indices[5000:]
tsne =  TSNE(n_components=2, random_state=42)
X_reduced_tsne = tsne.fit_transform(X_train[train_idx])
def plot_digits(X, y, title=None):
    # get center of each digit
    pd_center = pd.DataFrame({'y': y, 'x1': X[:, 0], 'x2': X[:, 1]}).groupby('y').median().reset_index()
    
    fig=plt.figure(figsize=(10,7))
    plt.scatter(X[:,0], X[:,1], c=y, cmap='Spectral', alpha=0.7, s=3, marker='o')
    for digit, x1, x2 in pd_center.to_numpy():
        plt.text(x1, x2, s=int(digit), fontdict={"weight": "bold", "size": 15})
    plt.colorbar(ticks=np.arange(10), boundaries=np.arange(11)-0.5)
    plt.axis("off")
    if title:
        plt.title(title, fontsize=18)
plot_digits(X_reduced_tsne, y_train[train_idx], 'T-SNE')<jupyter_output><empty_output><jupyter_text>UMAP1. fit umap to trainset and plot trainset embeddings, static and interactive for fun2. use embedding from step 1. to transform new data and plot testset embeddings3. repeat step 1 and 2 with supervised umap4. combine supervised umap with randomforest classifier, gridsearch both umap and rnd_clf hyperparameters<jupyter_code>import umap
import umap.plot<jupyter_output><empty_output><jupyter_text>Steps 1 & 2. fit on train then transform on test. Unsupervised<jupyter_code>unsup_umap = umap.UMAP(n_components=2, random_state=42)
X_reduced_unsup_umap = unsup_umap.fit_transform(X_train[train_idx])
plot_digits(X_reduced_unsup_umap, y_train[train_idx], title=f'Unsupervised UMAP on Trainset (size {len(train_idx)})')
X_unsup_umap_test = unsup_umap.transform(X_train[test_idx])
plot_digits(X_unsup_umap_test, y_train[test_idx], f'Unsupervised UMAP on Testset (size {len(test_idx)})')<jupyter_output><empty_output><jupyter_text>Interactive plots<jupyter_code>from bokeh.io import output_notebook
output_notebook()
hover_data = pd.DataFrame({'index': train_idx, 'digit': y_train[train_idx]})
p = umap.plot.interactive(unsup_umap, labels=y_train[train_idx], hover_data=hover_data, point_size=2, cmap='Spectral', width=400, height=400)
umap.plot.show(p)<jupyter_output><empty_output><jupyter_text>Steps 3. fit on train then transform on test. Supervised<jupyter_code>sup_umap = umap.UMAP(n_components=2, random_state=42)
X_train_sup_umap = sup_umap.fit_transform(X_train[train_idx], y=y_train[train_idx])
plot_digits(X_train_sup_umap, y_train[train_idx], title=f'Supervised UMAP on Trainset (size {len(train_idx)})')
X_test_sup_umap = sup_umap.transform(X_train[test_idx])
plot_digits(X_test_sup_umap, y_train[test_idx], f'Supervised UMAP on Testset (size {len(test_idx)})')<jupyter_output><empty_output><jupyter_text>UMAP with randomforest<jupyter_code>pipeline = Pipeline([
    ('umap', umap.UMAP(random_state=42)),
    ('rf_clf', RandomForestClassifier(min_samples_split=5, random_state=42))
])

param_grid = [{
    "umap__n_components": [2, 20],
    "rf_clf__n_estimators": [100, 200] 
}]

gridcv = GridSearchCV(pipeline, param_grid, n_jobs=4, cv=3, verbose=2)
gridcv.fit(X_train[train_idx], y_train[train_idx])
pd.DataFrame(gridcv.cv_results_).sort_values('rank_test_score')
accuracy_score(y_train[test_idx], gridcv.best_estimator_.predict(X_train[test_idx]))
time_a_model(gridcv.best_estimator_, (X_train, y_train), testset=(X_test, y_test), predict=True)<jupyter_output>Fitting Pipeline with sample shape (60000, 784) took 142.4 seconds.
Accuracy on test set: 0.963
{'fraction_non_alphanumeric': 0.12366192682537147, 'fraction_numerical': 0.03403099536667199, 'mean_word_length': 7.204456094364351, 'pattern_counts': {'":': 4, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '565507', 'license': 'Apache-2.0', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2375, 'n_words': 469, 'path': 'exercises/ch08.ipynb', 'repo_name': 'gie-de-moo/handson-ml2', 'size': '6259'}