<jupyter_start><jupyter_text>AutoEncoder<jupyter_code>import torch
import torchvision
from torch import nn
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.datasets import MNIST
from torchvision.utils import save_image
# torch.cuda.is_available()
device = torch.device("cuda:0")
# device

torch.cuda.set_device(0)

# torch.cuda.current_device()
# torch.cuda.device_count()

torch.cuda.get_device_name(torch.cuda.current_device())
# input_dim = X_train_counts.shape[1]
# encoding_dim = 32

input_dim = dimX.shape[1]
encoding_dim = 32
input_dim
num_epochs = 10000
batch_size = 2048
learning_rate = 1e-4

# data = spy_sparse2torch_sparse(X_train_counts)
# dataloader = DataLoader(data, batch_size=batch_size, shuffle=True)
# class autoencoder(nn.Module):
#     def __init__(self):
#         super(autoencoder , self).__init__()
        
#         self.encoder = nn.Sequential(
#             nn.Linear(input_dim , 512),
#             nn.ReLU(True),
#             nn.Linear(512 , 256),
#             nn.ReLU(True),
#             nn.Linear(256 , encoding_dim))
        
#         self.decoder = nn.Sequential(
#             nn.Linear(encoding_dim , 256),
#             nn.ReLU(True),
#             nn.Linear(256 , 512),
#             nn.ReLU(True),
#             nn.Linear(512 , input_dim),
#             nn.Tanh())
        
#     def forward(self , x):
#         x = self.encoder(x)
#         x = self.decoder(x)
#         return x
    
class autoencoder(nn.Module):
    def __init__(self):
        super(autoencoder , self).__init__()
        
        self.encoder = nn.Sequential(
            nn.Linear(input_dim , 64),
            nn.ReLU(True),
            nn.Dropout(p=0.2),
            nn.Linear(64 , encoding_dim))
        
        self.decoder = nn.Sequential(
            nn.Linear(encoding_dim , 64),
            nn.ReLU(True),
            nn.Dropout(p=0.2),
            nn.Linear(64 , input_dim),
            nn.Tanh())
        
    def forward(self , x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x
    
dimX=dimX.astype('float32')
import time
torch.rand(1).to(device)
model = autoencoder().cuda()
print(summary(model, (input_dim,) ))
historyloss=[]

from torchsummary import summary

criterion = nn.MSELoss()
optimizer = torch.optim.Adam(
    model.parameters() , lr = learning_rate , weight_decay=1e-5)
n = dimX.shape[0]
for epoch in range(num_epochs):
    i=0 
    for i in range(n//batch_size):
        t = time.time();
#         inpx = data.to_dense()
        inpx =  torch.from_numpy(dimX[i*batch_size : (i+1)*batch_size])
        inpx = Variable(inpx).cuda()#using gpu vram
        
        #-----------forward-----------
        output = model(inpx)
        loss = criterion(output , inpx)
        
        #-------------backward--------
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
#         if(i%10==0):
#             print("time pased {} to process ".format(time.time()-t ))
        i+=1
    #------------log-----------------
    if(epoch%50==0):
        print("epoch [{}/{}] , loss:{}".
              format(epoch+1 , num_epochs,loss))
    historyloss.append(loss)    
        
plt.plot(range(len(historyloss)) ,historyloss ,  )
torch.save(model.state_dict(), './sim_autoencoder.pth')
# device = torch.device("cpu")
model.cuda()
modelcpu = autoencoder().cpu()
modelcpu.load_state_dict(torch.load('./sim_autoencoder.pth'))

model.encoder.cuda()
# inputready = Variable(torch.from_numpy(dimX)).cuda()
# encoder = model.encoder.cuda()
data=modelcpu.encoder(torch.from_numpy(dimX)).detach().numpy()
data.shape
from sklearn.cluster import KMeans
nc=10
# losses = []
# for nc in range(50,100):
#     kmeans = KMeans(n_clusters=nc, random_state=0).fit(dimX)

#     losses.append((nc,kmeans.score(dimX)))
kmeans = KMeans(n_clusters=nc, random_state=0).fit(data)
kmeans.score(data)
plt.plot([x[0] for x in losses] , [x[1] for x in losses] , 'rp')
plt.show()<jupyter_output><empty_output><jupyter_text>visualization<jupyter_code>import random
from sklearn.manifold import TSNE
sample_n = 10000
l=random.sample(list(df.index),sample_n)

datasample = [df.iloc[x][df.columns[-1]] for x in l]
# pred=pipe.predict()
X_embedded = TSNE(n_components=2).fit_transform(
    modelcpu.encoder(torch.from_numpy(
    svd.transform(vectorizer.transform(datasample)).astype('float32')
    )).detach().numpy()
                 )
datapiped = modelcpu.encoder(torch.from_numpy(
    svd.transform(vectorizer.transform(datasample)).astype('float32')
    )).detach().numpy()
# pred=pipe.predict(datasample)
plt.figure(figsize=(20,10))
plt.scatter(X_embedded[:,0] , X_embedded[:,1]   )
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15755153554901136, 'fraction_numerical': 0.02103491796381994, 'mean_word_length': 2.761867088607595, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '223850', 'license': 'MIT', 'n_tokens_mistral': 1737, 'n_tokens_neox': 1653, 'n_words': 268, 'path': 'AdvancedClustering.ipynb', 'repo_name': 'sajjadGG/CovidSocial2020', 'size': '4754'}