<jupyter_start><jupyter_text>Initialization<jupyter_code>torch.cuda.init()
device = torch.device('cuda:1')
print(torch.cuda.get_device_name(device))
# class AutoEncoder(nn.Module):

#     def __init__(self):
#         super(AutoEncoder, self).__init__() 
        
#         #encode
#         self.e1 = nn.Linear(9216,6000)
#         self.e2 = nn.Linear(6000,4000)
# #         self.e3 = nn.Linear(4000,2000)
# #         self.e4 = nn.Linear(2000,1000)
        
#         #decode
# #         self.d1 = nn.Linear(1000,2000)
# #         self.d3 = nn.Linear(2000,4000)
#         self.d1 = nn.Linear(4000,6000)
#         self.d2 = nn.Linear(6000,9216)

#     def forward(self, x):
#         encode = self.e2(F.relu(self.e1(x)))
#         return self.d2(F.relu(self.d1(encode)))
class AutoEncoder(nn.Module):

    def __init__(self):
        super(AutoEncoder, self).__init__() 
        
        #encode
        self.e1 = nn.Linear(9216,4608)
        self.e2 = nn.Linear(4608,1024)
        
        #decode
        self.d1 = nn.Linear(1024,4608)
        self.d2 = nn.Linear(4608,9216)

    def forward(self, x):
        encode = self.e2(F.relu(self.e1(x)))
        
        return self.d2(F.relu(self.d1(encode)))
base_path = '/ssd2/chetanp'

epochs = 250
epochs0 = 4
batch_size = int((2 ** 15))
lr = 1e-5
lr0 = 1e-4
dataset = []
for i in range(4):
    path = os.path.join(base_path,'brown_e_base'+str(i)+'.pt')
    data = torch.load(path)
    dataset.append(TensorDataset(data))
dataset = ConcatDataset(dataset)
data_loader = DataLoader(dataset,batch_size = batch_size,num_workers = 16)
model = nn.DataParallel(AutoEncoder(),device_ids =[1,2]).to(device)
path = os.path.join(base_path,'brown_e_base_compressor_superfine.pth')
model.load_state_dict(torch.load(path))
criterion = nn.CosineSimilarity()
loss_function = lambda x,y: (1-criterion(x,y)).mean()

# loss_function = nn.MSELoss()
optimizer = torch.optim.Adam(
    model.parameters(), lr=lr0, weight_decay=1e-5)<jupyter_output><empty_output><jupyter_text>Training The Model<jupyter_code>for i in range(epochs0):
    
    running_loss = 0
    count = 0
    
    for data in tqdm(data_loader):
        ## Get Data
        sample = data[0].to(device)
        
        ## Pass forward
        output = model(sample)
        loss = loss_function(output,sample)
        
        ## Update Parameters
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        ## Update Running Loss
        running_loss += float(loss)
        count += 1
        
#         if(i == 0): print('Enough GPU Memory Present')
    print('Epoch ', str(i+1), ' Loss: ', str(running_loss/count))
optimizer = torch.optim.Adam(
    model.parameters(), lr=lr, weight_decay=1e-5)

path = os.path.join(base_path,'optimizer_superfine.pth')
optimizer.load_state_dict(torch.load(path))
lambda1 = lambda epoch: 0.98 ** epoch
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1)
losses = []
for i in range(epochs):
    
    running_loss = 0
    count = 0
    
    for data in tqdm(data_loader):
        ## Get Data
        sample = data[0].to(device)
        
        ## Pass forward
        output = model(sample)
        loss = loss_function(output,sample)
        
        ## Update Parameters
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        ## Update Running Loss
        running_loss += float(loss)
        count += 1
        
#         if(i == 0): print('Enough GPU Memory Present')
    print('Epoch ', str(i+1), ' Loss: ', str(running_loss/count))
    losses.append(running_loss/count)
for i in range(epochs):
    
    running_loss = 0
    count = 0  
    
    for data in tqdm(data_loader):
        ## Get Data
        sample = data[0].to(device)
        
        ## Pass forward
        output = model(sample)
        loss = loss_function(output,sample)
        
        ## Update Parameters
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        ## Update Running Loss
        running_loss += float(loss)
        count += 1
        
    print('Epoch ', str(i+1), ' Loss: ', str(running_loss/count))
    scheduler.step()
path = os.path.join(base_path,'brown_e_base_compressor_superfine.pth')
torch.save(model.state_dict(), path)

path = os.path.join(base_path,'optimizer_superfine.pth')
torch.save(optimizer.state_dict(), path)

path = os.path.join(base_path,'loss.csv')

import csv

with open(path, 'w') as myfile:
    wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
    wr.writerow(losses)
%reset -f<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13803614195514913, 'fraction_numerical': 0.035924232527759635, 'mean_word_length': 2.5778816199376946, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '492629', 'license': 'Apache-2.0', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1605, 'n_words': 275, 'path': 'Experiments/Compression/Implement_AutoEncoder.ipynb', 'repo_name': 'jsedoc/ConceptorDebia', 'size': '4593'}