<jupyter_start><jupyter_text>Activity 1: Loading and exploring the digits dataset<jupyter_code>from sklearn import datasets
from IPython.display import display, Markdown<jupyter_output><empty_output><jupyter_text>digits.images and digits.targets<jupyter_code>digits = datasets.load_digits()
digits.images[0]
scaled_images = ((1-digits.images/16)*255).astype('uint8') # Scaling pixels between 0 (black) and 255 (white)
scaled_images[0]
from PIL import Image
for i in range(5):
    display(Image.fromarray(scaled_images[i]).resize((100,100)))
digits.target[:5]
from collections import Counter
Counter(digits.target) #Number of each class in the dataset<jupyter_output><empty_output><jupyter_text>digits.data<jupyter_code>digits.data.shape<jupyter_output><empty_output><jupyter_text>Activity 2, Variance Thresholding<jupyter_code>digits.images.var(axis=0).round(1)
digits.data.var(axis=0).round(1) # The same values as above but flattened from an 8x8 image to a 64 dimensional vector
digits.images.var(axis=0).round(1) < 1 # Pixels with variance < 1
from sklearn.feature_selection import VarianceThreshold
variance_threshold = VarianceThreshold(threshold = 1.)
variance_threshold
new_data = variance_threshold.fit_transform(digits.data)
print(f'Previous shape:{digits.data.shape}, new shape: {new_data.shape}')<jupyter_output>Previous shape:(1797, 64), new shape: (1797, 48)<jupyter_text>Activity 3, Finding Covariance in the Digits Dataset<jupyter_code>import numpy as np
cor = np.corrcoef(digits.data, rowvar=False) 
print(cor[:3, :3]) # we see wherever there is a pixel with dimensionality zero
print(cor.shape)
cor_abs = np.abs(cor)
n = cor.shape[0]
for i in range(n):
    for j in range(n):
        if i == j:
            continue
        if cor_abs[i, j] > 0.9:
            print(f'Pixel {i} and {j} have correlation: {cor[i,j]:.2f}')<jupyter_output>Pixel 2 and 58 have correlation: 0.94
Pixel 58 and 2 have correlation: 0.94<jupyter_text>Activity 4, An Introduction to Word Embeddings<jupyter_code>import spacy #pip install spacy

nlp = spacy.load('en_core_web_md') #python -m spacy download en_core_web_md
tokens = nlp(u'dog cat banana')
for token1 in tokens:
    for token2 in tokens:
        print(token1.text, token2.text, token1.similarity(token2))<jupyter_output>dog dog 0.99999994
dog cat 0.80168545
dog banana 0.24327643
cat dog 0.80168545
cat cat 1.0000001
cat banana 0.28154364
banana dog 0.24327643
banana cat 0.28154364
banana banana 1.0000001<jupyter_text>Activity 5, Random Projections<jupyter_code>from random import seed
seed(100)
import numpy as np
X = np.random.rand(100, 10000) # Generate a large random matrix.
print(X.shape)
from sklearn.random_projection import SparseRandomProjection
sparse_random_projection = SparseRandomProjection()
X_reduced = sparse_random_projection.fit_transform(X)
X_reduced.shape
sparse_random_projection.components_<jupyter_output><empty_output><jupyter_text>Activity 6, PCA and sklearn<jupyter_code>from sklearn import decomposition
pca = decomposition.PCA(n_components = 2)
pca.fit(digits.data)
pca.components_.shape<jupyter_output><empty_output><jupyter_text>The principal components<jupyter_code>c1, c2 = pca.components_.reshape(-1, 8, 8)
c1.round(1) #the weight of each pixel to the first principal component
c2.round(1) #the weight of each pixel to the second principal component<jupyter_output><empty_output><jupyter_text>Explained Variance<jupyter_code>pca.explained_variance_ratio_ # the first PC explains 14.9% of the variance, the next 13.6%
pca.explained_variance_ratio_.sum() # this means 28.5% of total variance is explained by the first two principal components
all_variance = decomposition.PCA().fit(digits.data).explained_variance_ratio_
from matplotlib import pyplot
print(all_variance.cumsum().round(2))
display(Markdown("#### Cumulative variance explained by principal component on the digits dataset"))
display(pyplot.bar(x = range(len(all_variance)), height=all_variance.cumsum()))
from matplotlib import pyplot
princomps  = pca.transform(digits.data)
fig, ax = pyplot.subplots(figsize = (15,15))
ax.scatter(princomps[:,0], princomps[:,1], c=digits.target)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10427679000480539, 'fraction_numerical': 0.04613166746756367, 'mean_word_length': 6.680811808118081, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '443835', 'license': 'MIT', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1423, 'n_words': 355, 'path': 'CH4/Chapter 4 Exercises.ipynb', 'repo_name': 'doc-E-brown/Applied-Unsupervised-Learning-with-Python', 'size': '4162'}