<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_openml
from sklearn.utils import check_random_state
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Curse of Dimensionality<jupyter_code>distances = []
dimensions = range(1,10000,250)
for i in dimensions:
  n_dimensions = i 
  point1 = np.random.rand(n_dimensions)
  point2 = np.random.rand(n_dimensions)
  distances.append(np.linalg.norm(point1-point2))
fig, ax = plt.subplots()
ax.plot(dimensions, distances, "b-", label = 'Euclidean Distance')
ax.set_xlabel("Dimensions")
ax.set_ylabel("Distance")
plt.legend(loc="upper left", fontsize=8)
plt.title('Curse of Dimensionality')
plt.show()<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>z = np.linspace(0, 3*np.pi)
x = np.cos(z - np.pi/2)
y = np.sin(z - np.pi/2)

X = np.array(list(zip(x,y,z)))
fig, ax = plt.subplots(subplot_kw=dict(projection='3d'))
ax.scatter3D(x, y, z)
plt.show()
X_centered = X - X.mean(axis=0) 
U, s, Vt = np.linalg.svd(X_centered) 
c1 = Vt.T[:, 0] 
c2 = Vt.T[:, 1]

# Draw plane
xx, yy = np.meshgrid(np.arange(np.min(X_centered[:,0]), np.max(X_centered[:,0]), .01), np.arange(np.min(X_centered[:,1]), np.max(X_centered[:,1]), .01))

z1 = (-c1[0] * xx - c2[1] * yy ) * 1. / c1[2]
z2 = (-c2[0] * xx - c2[1] * yy ) * 1. / c2[2]
# plot the surface
plt3d = plt.figure().gca(projection='3d')
plt3d.plot_surface(xx, yy, z1, alpha=0.09)
plt3d.plot_surface(xx, yy, z2, alpha=0.09)
plt3d.scatter(*(X_centered.T), c = X_centered[:,0], cmap='inferno', label = 'Projection')

plt.show()
W2 = Vt.T[:, :2] 
X2D = X_centered.dot(W2)
fig, ax = plt.subplots()
plt.title('2D projection')
ax.scatter(X2D[:,0],X2D[:,1], c = X2D[:,1], cmap='inferno', label = 'Projection') 
ax.set_xlabel("X")
ax.set_ylabel("Y")
plt.show()<jupyter_output><empty_output><jupyter_text>Compression<jupyter_code># Load data from https://www.openml.org/d/554
X, y = fetch_openml('mnist_784', version=1, return_X_y=True)
X = X / 255.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=10000, random_state = 123)
n_rows = 4
n_cols = 10
plt.figure(figsize=(n_cols * 1.2, n_rows * 1.2))
random_state = check_random_state(0)
permutation = random_state.permutation(X_train.shape[0])
X_train = X_train[permutation]
y_train = y_train[permutation]
for row in range(n_rows):
    for col in range(n_cols):
        index = n_cols * row + col
        plt.subplot(n_rows, n_cols, index + 1)
        plt.imshow(X_train[index].reshape(28,28), cmap="binary", interpolation="nearest")
        plt.axis('off')
        plt.title(y_train[index], fontsize=12)
plt.subplots_adjust(wspace=0.2, hspace=0.5)
plt.show()
import timeit

from sklearn.linear_model import Perceptron

clf = Perceptron(tol=1e-3, verbose = 0, random_state=123)

clf.fit(X, y)
%%timeit
clf.fit(X, y)
clf.score(X, y)
from sklearn.decomposition import PCA

pca = PCA(n_components=784)
pca.fit(X)
print(pca.explained_variance_ratio_)
from sklearn.decomposition import PCA
pca = PCA(n_components=.95)
pca.fit(X)
X_compressed = pca.transform(X)
X_compressed.shape
clf = Perceptron(tol=1e-3, verbose = 0, random_state=123)

clf.fit(X_compressed, y)
%%timeit
clf.fit(X_compressed, y)
clf.score(X_compressed, y)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>from sklearn.manifold import TSNE

X_compressed = PCA(n_components=30).fit_transform(X[:10000])
X2D = TSNE(n_components=2, perplexity=40, n_iter=1000).fit_transform(X_compressed)
fig, ax = plt.subplots(figsize=(15,10))
scatter = ax.scatter(X2D[:,0],X2D[:,1], c = np.array(y[:10000], dtype = 'int'), marker='o', cmap = 'tab10', label=np.unique(y[:10000]), s=30)
legend1 = ax.legend(*scatter.legend_elements(),
                    loc="upper left", title="Classes")
ax.add_artist(legend1)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17134036532029842, 'fraction_numerical': 0.043220993053768975, 'mean_word_length': 5.7617391304347825, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '554238', 'license': 'MIT', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1658, 'n_words': 286, 'path': "Roberto_Zerbini's_Blog_Dimensionality_Reduction.ipynb", 'repo_name': 'robertozerbini/blog', 'size': '3887'}