<jupyter_start><jupyter_text>Unzip file<jupyter_code>os.chdir("/content/drive/Shareddrives/Documentation/Data/bm/")
!gunzip /content/drive/Shareddrives/Documentation/Data/bm/rna_scale_bmcite.csv.gz

from google.colab import drive
drive.mount('/content/drive')
import os
os.chdir("/content/drive/Shareddrives/Documentation/Jiaxin_Li/")<jupyter_output><empty_output><jupyter_text>Import<jupyter_code>import sys
import argparse

import torch
from torch import nn
import torch.distributed as dist
import torch.backends.cudnn as cudnn
from torchvision import datasets
from torchvision import transforms as pth_transforms
from torchvision import models as torchvision_models
import pandas as pd
from torch.utils.data import Dataset
from functools import partial
import numpy as np
from torch import optim
from pathlib import Path
from torch.autograd import Variable

import os
import argparse
import os
import sys
import datetime
import time
import math
import json
from pathlib import Path

import numpy as np
from PIL import Image
import torch
import torch.nn as nn
import torch.distributed as dist
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
from torch import optim
from torchvision import datasets, transforms
from torchvision import models as torchvision_models
import math
from functools import partial

import torch
import torch.nn as nn


import torch.nn.functional as F
from torch import optim
from torch.autograd import Variable<jupyter_output><empty_output><jupyter_text>Set seed<jupyter_code>generator=torch.Generator().manual_seed(42)
import utils
import vision_transformer_sc as vits
from utils import trunc_normal_<jupyter_output><empty_output><jupyter_text>Re import files<jupyter_code>import os
os.chdir("/content/drive/Shareddrives/Documentation/Jiaxin_Li/")
from importlib import reload  
import vision_transformer_sc as vits  
vits = reload(vits)<jupyter_output><empty_output><jupyter_text>Read in Expr and meta<jupyter_code>expr = pd.read_csv("/content/drive/MyDrive/scRNA/data/rna_withlabels.csv", index_col=0)
meta = pd.DataFrame(expr["celltype.l1.5"])
meta
meta["perturb"] = meta["celltype.l1.5"]
expr = expr.drop("celltype.l1.5", axis = 1)
expr = expr.T<jupyter_output><empty_output><jupyter_text>Dataset Class<jupyter_code>class scRNACSV(Dataset):
  def __init__(self, expr, meta, transform = None, target_transform=None):
    self.expr = expr
    self.meta = meta
    self.cells = list(expr.columns)
    self.labels = list(meta["perturb"])
    self.samples = [(self.cells,self.labels[i]) for i in range(len(self.labels))]
    self.label_keys = list(set(self.labels))
    self.label_keys.sort()
    self.label_dic = {}
    for label, i in zip(self.label_keys, range(len(self.label_keys))):
      self.label_dic[label] = i
    print(f"This is the label dictionary of this dataset {self.label_dic}")
    self.transform = transform
    self.target_transform = target_transform
  def __len__(self):
    return self.expr.shape[1]

  def __getitem__(self,idx):
    one_cell = torch.from_numpy(np.array(self.expr.iloc[:,idx]))
    if self.transform:
      ret = self.transform(one_cell)
    else:
      ret = one_cell
    lab = self.label_dic[self.labels[idx]]
    
    return ret, self.label_dic[self.labels[idx]]
 class DINO_Crop(object):
  def __init__(self, global_crops_scale, local_crops_scale, local_crops_number):
    self.global_crops_scale = global_crops_scale
    self.local_crops_scale = local_crops_scale
    self.local_crops_number = local_crops_number
    
  def __call__(self, x):
    inputs = []
    data = x  
    length = x.shape[0]
    #print("This is the length")
    #print(length)
    global_size = int(length * self.global_crops_scale)
    global_index_1 = np.random.choice(length, size = global_size, replace = False)
    
    global_corr_1 = torch.from_numpy(global_index_1)
    
    global_data_1 = data[global_index_1,]
    global_input_1 = torch.cat([global_data_1, global_corr_1]).float()
    return global_input_1<jupyter_output><empty_output><jupyter_text>Initiate Crop<jupyter_code>crop = DINO_Crop(0.5,0.25,8)
dataset = scRNACSV(expr, meta, transform = crop)<jupyter_output>This is the label dictionary of this dataset {'B cell': 0, 'CD4 T': 1, 'CD8 T': 2, 'MAIT': 3, 'Mono/DC': 4, 'NK': 5, 'Progenitor cells': 6, 'gdT': 7}<jupyter_text>Split dataset<jupyter_code>train_length = int(len(dataset) * 0.8)
test_length = len(dataset) - train_length
train_dataset, test_dataset = torch.utils.data.random_split(dataset=dataset, lengths = [train_length, test_length], generator=torch.Generator().manual_seed(42))<jupyter_output><empty_output><jupyter_text>Dataloader<jupyter_code>train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=16, shuffle = True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16, shuffle = True)<jupyter_output><empty_output><jupyter_text>Supervised Train<jupyter_code>def train_one_epoch(trainloader, simple_model, epoch, optimizer):
  running_loss = 0.0
  simple_model.train()
  lossF = nn.CrossEntropyLoss()
  sum_loss = 0
  num = 0
  for batch_idx, (data, label) in enumerate(trainloader):
    data, label = Variable(data).cuda(), Variable(label).cuda()
    #data, label = Variable(data), Variable(label)
    optimizer.zero_grad()
    output = simple_model(data)
    loss = lossF(output, label)
    sum_loss += loss
    num += 1
    loss.backward()
    optimizer.step()
    #plot_grad_flow(simple_model.named_parameters())
    if batch_idx % 100 == 0:
      print("Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss {:.6f}".format(
                epoch, batch_idx*len(data), len(trainloader.dataset),
                100.*batch_idx/len(trainloader), loss.item()))
  loss_stat = (sum_loss/num).detach().cpu().numpy().item()
  log_stats = {**{f'Average Loss': loss_stat}, 'epoch': epoch + 1}
  with (Path(output_dir) / "loss.txt").open("a") as f: f.write(json.dumps(log_stats) + "\n")
os.chdir("/content/drive/Shareddrives/Documentation/Jiaxin_Li/Supervised_perturb/")<jupyter_output><empty_output><jupyter_text>Train loop<jupyter_code>os.chdir("/content/drive/Shareddrives/Documentation/Jiaxin_Li/Supervised_perturb/")
!rm -r ./128+128_bmcite_scale_Aug31_M
os.chdir("/content/drive/Shareddrives/Documentation/Jiaxin_Li/Supervised_perturb/")
output_dir = './128+128_bmcite_scale_Aug31_M' ### customized your own output_dir
model = vits.vit_tiny()
optimizer = optim.Adam(model.parameters(), lr = 1e-4)
model.cuda()
Path(output_dir).mkdir(parents=True, exist_ok=True)
saveckp_freq = 1
to_restore = {"epoch": 0}
utils.restart_from_checkpoint(
    os.path.join(output_dir, "checkpoint.pth"),
    run_variables=to_restore,
    model=model,
    optimizer=optimizer,
)
start_epoch = to_restore["epoch"]
for epoch in range(start_epoch, 1000):
    model.train()
    train_one_epoch(train_loader, model, epoch, optimizer)
    # ============ writing logs ... ============
    save_dict = {
        'model': model.state_dict(),
        'optimizer': optimizer.state_dict(),
        'epoch': epoch + 1,
    }
    # Train loop
    if epoch % 3 == 0:
      correct, total = 0, 0
    
      with torch.no_grad():
        # Iterate over the test data and generate predictions
          for i, data in enumerate(train_loader, 0):
          # Get inputs
            inputs, targets = data
            inputs = inputs.cuda()
            targets = targets.cuda()
          # Generate outputs
            model.eval()
            outputs = model(inputs)
          # Set total and correct
            _, predicted = torch.max(outputs.data, 1)
            #print(predicted)
            #print(outputs)
            total += targets.size(0)
            correct += (predicted == targets).sum().item()
            # Print accuracy
          train_acc = "%f %%" %(100 * correct / total)
          print(train_acc)
    # Testing loop
      correct, total = 0, 0
      with torch.no_grad():
        # Iterate over the test data and generate predictions
          for i, data in enumerate(test_loader, 0):
          # Get inputs
            inputs, targets = data
            inputs = inputs.cuda()
            #print(inputs.shape)
            #print(inputs)
            targets = targets.cuda()
          # Generate outputs
            model.eval()
            outputs = model(inputs)
          # Set total and correct
            _, predicted = torch.max(outputs.data, 1)
            #print(inputs[:,0:20])
            print(predicted)
            #print(outputs[:,0:10])
            total += targets.size(0)
            correct += (predicted == targets).sum().item()
        # Print accuracy
          test_acc =  "%f %%" %(100 * correct / total)
          print(test_acc)
      acc_stats = {"Training Accuracy": train_acc, "Testing Accuracy":  test_acc, "epoch": epoch+1}
      with (Path(output_dir) / "acc.txt").open("a") as f: f.write(json.dumps(acc_stats) + "\n")
    utils.save_on_master(save_dict, os.path.join(output_dir, 'checkpoint.pth'))
    if saveckp_freq and epoch % saveckp_freq == 0:
        utils.save_on_master(save_dict, os.path.join(output_dir, f'checkpoint{epoch:04}.pth'))<jupyter_output>Train Epoch: 0 [0/24537 (0%)]	Loss 2.135148
Train Epoch: 0 [1600/24537 (7%)]	Loss 2.016843
Train Epoch: 0 [3200/24537 (13%)]	Loss 1.872496
Train Epoch: 0 [4800/24537 (20%)]	Loss 2.073593
Train Epoch: 0 [6400/24537 (26%)]	Loss 1.548345
Train Epoch: 0 [8000/24537 (33%)]	Loss 1.440009
Train Epoch: 0 [9600/24537 (39%)]	Loss 1.127157
Train Epoch: 0 [11200/24537 (46%)]	Loss 1.466436
Train Epoch: 0 [12800/24537 (52%)]	Loss 1.150375
Train Epoch: 0 [14400/24537 (59%)]	Loss 1.081950
Train Epoch: 0 [16000/24537 (65%)]	Loss 1.034737
Train Epoch: 0 [17600/24537 (72%)]	Loss 1.039242
Train Epoch: 0 [19200/24537 (78%)]	Loss 0.907865
Train Epoch: 0 [20800/24537 (85%)]	Loss 0.765902
Train Epoch: 0 [22400/24537 (91%)]	Loss 0.739188
Train Epoch: 0 [24000/24537 (98%)]	Loss 0.873511
75.991360 %
tensor([4, 6, 1, 1, 1, 4, 4, 1, 2, 0, 2, 6, 1, 6, 0, 1], device='cuda:0')
tensor([1, 1, 1, 1, 2, 0, 4, 2, 4, 1, 6, 1, 2, 6, 6, 2], device='cuda:0')
tensor([1, 4, 4, 4, 1, 1, 0, 5, 4, 1, 2, 1, 1, 4, 4, 4], device='cud[...]
{'fraction_non_alphanumeric': 0.1292813304296179, 'fraction_numerical': 0.04741635319738666, 'mean_word_length': 3.885396518375242, 'pattern_counts': {'":': 4, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '221293', 'license': 'MIT', 'n_tokens_mistral': 3888, 'n_tokens_neox': 3501, 'n_words': 806, 'path': 'Supervised_bmcite_scale_Michelle.ipynb', 'repo_name': 'JiaxinLi-lipluszn/Organized-Notebook', 'size': '10102'}