<jupyter_start><jupyter_text>Umap acceleration using Rapids UMAP is a powerful dimensionality reduction tool that can benefit from GPU acceleration.<jupyter_code>import os
import cudf
import umap

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from sklearn.manifold.t_sne import trustworthiness
from cuml.manifold.umap import UMAP as cumlUMAP

%matplotlib inline
sns.set(style='white', rc={'figure.figsize':(25, 12.5)})

print('cuDF:', cudf.__version__)
print('pandas:', pd.__version__)
print('umap:', umap.__version__)
dataset_folder = 'fashion_mnist'
if not os.path.exists(dataset_folder):
    !git clone https://github.com/zalandoresearch/fashion-mnist.git fashion_mnist
# https://github.com/zalandoresearch/fashion-mnist/blob/master/utils/mnist_reader.py
def load_mnist(path, kind='train'):
    import os
    import gzip
    import numpy as np

    """Load MNIST data from `path`"""
    labels_path = os.path.join(path,
                               '%s-labels-idx1-ubyte.gz'
                               % kind)
    images_path = os.path.join(path,
                               '%s-images-idx3-ubyte.gz'
                               % kind)

    with gzip.open(labels_path, 'rb') as lbpath:
        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,
                               offset=8)

    with gzip.open(images_path, 'rb') as imgpath:
        images = np.frombuffer(imgpath.read(), dtype=np.uint8,
                               offset=16).reshape(len(labels), 784)

    return images, labels
dataset = os.path.join(dataset_folder, 'data/fashion')
print('Loading dataset:', dataset)
train, train_labels = load_mnist(dataset, kind='train')
test, test_labels = load_mnist(dataset, kind='t10k')
data = np.array(np.vstack([train, test]), dtype=np.float64) / 255.0
target = np.array(np.hstack([train_labels, test_labels]))
f"Train shape: {train.shape} and Test Shape: {test.shape}"<jupyter_output><empty_output><jupyter_text>Render sample image<jupyter_code>pixels = train[0].reshape((28, 28))
plt.imshow(pixels, cmap='gray')
%%time

record_data = (('fea%d'%i, data[:, i]) for i in range(data.shape[1]))
gdf = cudf.DataFrame(record_data)

gdf.head(3)
# hide warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)

%%timeit
g_embedding = cumlUMAP(n_neighbors=5, init='spectral').fit_transform(gdf)
%%timeit
embedding = umap.UMAP(n_neighbors=5, init="spectral").fit_transform(data)
classes = [
    'T-shirt/top',
    'Trouser',
    'Pullover',
    'Dress',
    'Coat',
    'Sandal',
    'Shirt',
    'Sneaker',
    'Bag',
    'Ankle boot']
embedding = umap.UMAP(n_neighbors=5, init="spectral").fit_transform(data)
g_embedding = cumlUMAP(n_neighbors=5, init="spectral").fit_transform(gdf)

g_embedding_numpy = g_embedding.to_pandas().values #it is necessary to convert to numpy array to do the visual mapping

fig, ax = plt.subplots(1, figsize=(14, 10))
plt.scatter(g_embedding_numpy[:,1], g_embedding_numpy[:,0], s=0.3, c=target, cmap='Spectral', alpha=1.0)
plt.setp(ax, xticks=[], yticks=[])
cbar = plt.colorbar(boundaries=np.arange(11)-0.5)
cbar.set_ticks(np.arange(10))
cbar.set_ticklabels(classes)
plt.title('Fashion MNIST Embedded via cumlUMAP');
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(20, 10))
ax[0].scatter(g_embedding_numpy[:,1], g_embedding_numpy[:,0], s=0.3, c=target, cmap='Spectral', alpha=1.0)
im = ax[1].scatter(embedding[:,1], embedding[:,0], s=0.3, c=target, cmap='Spectral', alpha=1.0)
ax[0].set_title('Fashion MNIST Embedded via cumlUMAP');
ax[1].set_title('Fashion MNIST Embedded via UMAP');

fig.subplots_adjust(right=0.8)
cax,kw = mpl.colorbar.make_axes([a for a in ax.flat])
cbar = plt.colorbar(im, cax=cax, **kw)
cbar.set_ticks(np.arange(10))
cbar.set_ticklabels(classes)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14889753566796368, 'fraction_numerical': 0.019974059662775615, 'mean_word_length': 4.377963737796374, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '252234', 'license': 'Apache-2.0', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1430, 'n_words': 278, 'path': 'umap_fashionist.ipynb', 'repo_name': 'martin-fabbri/rapids-hpc', 'size': '3855'}