<jupyter_start><jupyter_text>t-SNE on MNIST Handwritten Digits Dataset<jupyter_code># import required modules
import numpy as np # for matrix multiplication
import matplotlib.pyplot as plt # for plotting
import seaborn as sns # for plotting
import pandas as pd             # for data handling
from sklearn.preprocessing import StandardScaler # for column standardization
from sklearn.manifold import TSNE # for t-SNE
import time # for time measurement
import multiprocessing # for threading
import imageio # for GIF creation
print(multiprocessing.cpu_count())
# Read dataset
d0 = pd.read_csv('./../../AAIC-Course/datasets/mnist-digits-dataset/train.csv')
d0.head()
# split the dataset into features and labels
d_labels_all = d0['label']
d_data_all = d0.drop('label',axis=1)
print(d_labels_all.shape,d_data_all.shape)
def genTSNEGif(std_data, ndp, p, itr_list, img_name):
    '''
    std_data - Column Standardized Data
    ndp - Number of Data Points to consider in std_data
    p - Perplexity
    itr_list - List of iterations, each iteration will be a frame in GIF
    img_name - Name of GIF image
    '''
    
    print('No.Of Data Points - {0}, Perplexity - {1}, Iterations - {2}, ImageName - {3}'.format(
         ndp, p, itr_list, img_name))
    
    # list to hold the frames
    frames = []
    
    p_data = standardized_data[0:ndp,:]
    p_labels = d_train_labels[0:ndp]
    #print('t-SNE Data Points {0} and its Labels {1}'.format(p_data.shape, p_labels.shape))
    
    for itr_val in itr_list:
        time_start = time.time()
        
        model = TSNE(n_components=2,random_state=0,perplexity=p,n_iter=itr_val)
        
        tsne_data = model.fit_transform(p_data)
        time_elapsed = time.time() - time_start
        #print('t-SNE done! Time elapsed: {} seconds'.format(time.time() - time_start))
        
        tsne_data = np.vstack((tsne_data.T,p_labels)).T
        #print(tsne_data.shape)
        #tsne_data[:4]
        
        tsne_df = pd.DataFrame(tsne_data,columns=['Dim_1','Dim_2','label'])
        #tsne_df.head()

        g = sns.FacetGrid(tsne_df,hue='label',height=8).map(plt.scatter, 'Dim_1', 'Dim_2').add_legend();
        
        g.fig.suptitle('ndp={0} p={1} itr={2}'.format(ndp, p, itr_val))
        g.fig.canvas.draw()
        
        image = np.frombuffer(g.fig.canvas.tostring_rgb(), dtype='uint8')
        image  = image.reshape(g.fig.canvas.get_width_height()[::-1] + (3,))

        frames.append(image)
    
    kwargs_write = {'fps':1.0, 'quantizer':'nq'}
    imageio.mimsave('./{}.gif'.format(img_name), frames, fps=1)
    
    return<jupyter_output><empty_output><jupyter_text>Training Size - 15000<jupyter_code># Taking 15000 images for training
training_size = 15000

d_train_labels = d_labels_all.head(training_size)
d_train_data = d_data_all.head(training_size)
print('Shape of Training Data {0}, Label {1}'.format(d_train_data.shape, d_train_labels.shape) )

# standardize the data
standardized_data = StandardScaler().fit_transform(d_train_data.astype(np.float64))
standardized_data.shape<jupyter_output>Shape of Training Data (15000, 784), Label (15000,)<jupyter_text>t-SNE over 1000 data points<jupyter_code>genTSNEGif(standardized_data, 1000, 30, range(1000,10001,1000), 't_15000_d_1000_p_30')
genTSNEGif(standardized_data, 1000, 50, range(1000,10001,1000), 't_15000_d_1000_p_50')<jupyter_output>No.Of Data Points - 1000, Perplexity - 50, Iterations - [1000, 2000, 3000, 4000, 5000, 6000], ImageName - t_15000_d_1000_p_50<jupyter_text>t-SNE over 5000 data points<jupyter_code>genTSNEGif(standardized_data, 5000, 30, range(1000,10001,1000), 't_15000_d_5000_p_30')
genTSNEGif(standardized_data, 5000, 50, range(1000,10001,1000), 't_15000_d_5000_p_30')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12526709401709402, 'fraction_numerical': 0.06169871794871795, 'mean_word_length': 4.102179836512262, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '79620', 'license': 'MIT', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1368, 'n_words': 275, 'path': 'hands-on/t-SNE/MNIST-Digits-Dataset/.ipynb_checkpoints/t-SNE-MNIST-Digits-Dataset-Threaded-checkpoint.ipynb', 'repo_name': 'gymk/AAIC-Course', 'size': '3744'}