<jupyter_start><jupyter_text><jupyter_code>import os
import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(f'Using {device} device')
input_1 = torch.load('/content/drive/MyDrive/NLP/Trance/TENSOR/input_1')
input_2 = torch.load('/content/drive/MyDrive/NLP/Trance/TENSOR/input_2')
outputs = torch.load('/content/drive/MyDrive/NLP/Trance/TENSOR/outputs')
input_1.shape, input_2.shape, outputs.shape
input_1 = torch.split(input_1, 512)
input_2 = torch.split(input_2, 512)
outputs = torch.split(outputs, 512)

input_1[0].shape
n = int(len(input_1)*0.9)
input_1 = input_1[:n]
input_2 = input_2[:n]
outputs = outputs[:n]
encoder_1 = nn.Sequential(
    nn.Linear(input_1[0].shape[1], 512),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(512, 256),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(256, 64),
    nn.ReLU(),
    nn.Dropout(0.4),
)

encoder_2 = nn.Sequential(
    nn.Linear(input_2[0].shape[1], 512),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(512, 256),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(256, 64),
    nn.ReLU(),
    nn.Dropout(0.4),
)

decoder = nn.Sequential(
    nn.Linear(64, 256),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(256, 512),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(512, outputs[0].shape[1]),
    nn.ReLU()
)

class Model(nn.Module):
    def __init__(self, encoder_1, encoder_2, decoder):
        super(Model, self).__init__()
        self.encoder_1 = encoder_1
        self.encoder_2 = encoder_2
        self.decoder = decoder

    def forward(self, x_1, x_2):
        x_1 = self.encoder_1(x_1)
        x_2 = self.encoder_2(x_2)
        x = x_1 - x_2
        x = decoder(x)
        return x
class RMSELoss(torch.nn.Module):
    def __init__(self):
        super(RMSELoss,self).__init__()

    def forward(self,x,y):
        criterion = nn.MSELoss()
        loss = torch.sqrt(criterion(x*1000, y))
        return loss
# create a model from `AE` autoencoder class
# load it to the specified device, either gpu or cpu
model = Model(encoder_1, encoder_2, decoder).to(device)

# create an optimizer object
# Adam optimizer with learning rate 1e-3
optimizer = torch.optim.SGD(model.parameters(), lr=1e-2)

# mean-squared error loss
criterion = RMSELoss()
model.to(device)
# input_1 = input_1.to(device)
# input_2 = input_2.to(device)
# outputs = outputs.to(device)
from tqdm.notebook import tqdm
EPOCHS = 75000
for epoch in tqdm(range(EPOCHS)):
    running_loss = 0
    ctr = 0
    for x_1, x_2, o in zip(input_1, input_2, outputs):
        ctr += 1
        optimizer.zero_grad()
        x_1 = x_1.to(device)
        x_2 = x_2.to(device)
        o = o.to(device)
        pred = model(x_1, x_2)
        loss = criterion(pred, o)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()
    if epoch % 100 == 0:
        running_loss = running_loss/(len(input_1)*100)
        print(running_loss)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14558629776021081, 'fraction_numerical': 0.05006587615283267, 'mean_word_length': 3.1096075778078482, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '209277', 'license': 'MIT', 'n_tokens_mistral': 1257, 'n_tokens_neox': 1162, 'n_words': 232, 'path': 'SimpleLinear.ipynb', 'repo_name': 'saurabh1mohite/TDVR', 'size': '3036'}