<jupyter_start><jupyter_text>Train and create a submission modelSet up your variables below (make sure they point to the right place) and hit run. Let the model do it's thingRemember to use a GPU Runtime!<jupyter_code>!pip install python-dotenv
!pip install colab-env --upgrade
!pip install pytorch_lightning
from google.colab import drive
drive.mount('/content/drive')
import colab_env

colab_env.__version__
!more /content/drive/MyDrive/vars.env
import os

TEN_PERCENT_MODEL_PATH = os.getenv("TENPERCENT_MODEL_PATH")
patience = int(os.getenv("patience"))
WORKSPACE_PATH = os.getenv("WORKSPACE_PATH")

OUTPUT_MODEL_PATH = os.getenv("OUTPUT_MODEL_PATH")
print(OUTPUT_MODEL_PATH)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import cv2
import os 
from tqdm import tqdm

import torch
from torch import nn 
from torch.utils.data import Dataset, DataLoader
import torchvision
import torchvision.models as models
import torchvision.transforms as transforms
from torchsummary import summary

from torch.nn import Linear
from torch.nn import ReLU
from torch.nn import Sigmoid
from torch.nn import Module
from torch.optim import Adam
from torch.nn import L1Loss
from torch.nn.init import kaiming_uniform_
from torch.nn.init import xavier_uniform_

from torchvision.io import read_image
from google.colab.patches import cv2_imshow


from pathlib import Path
Path.ls = lambda x: list(x.iterdir())
# Config Class
class Config:
    def __init__(self):
        self.FOLDS = 5
        self.EPOCHS = 30
        self.DEVICE = 'cuda'
        self.TRAIN_BS = 16
        self.VALID_BS = 8
        self.loss_fn = nn.L1Loss()

config = Config()
path = Path(WORKSPACE_PATH)
path.ls()<jupyter_output><empty_output><jupyter_text>Data Prep & Loading<jupyter_code># get train 
train_df = pd.read_csv(str(path.ls()[5]))
test_df = pd.read_csv(str(path.ls()[3]))
test_df.head()
# unique patinet ids 
unique_patient_ids = train_df.id.unique()
len(unique_patient_ids)
unique_patient_ids<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>train_df.columns
train_cols_to_drop = ['id' , 'usage' ,'image_file' ,'image_rename' ,'current_directory' ,'layer' ,'img_type' ,'N' ,'Data' ,'id_encoding' ,'image_rename_encoding', 'OS', 'DDN', 'Date biopsie']
class Dataset:
    def __init__(self, path, df, folder = 'Train', is_train = 1):
      """Summary or Description of the Function
      
      Parameters:
      path (Path): path of drive folder that contains csvs and train / test image folders 
      df (Pandas df )  : sub fold dataframe for fold patients 
      folder (str) : train or test folder 
      
      Returns:
      img , tab and target 
      
      """

      self.df = df
      self.folder = folder
      self.path = path 
      self.is_train = is_train
      
      self.train_transform = transforms.Compose([
                transforms.ToPILImage(),
                transforms.Resize((224, 224)),
                transforms.RandomHorizontalFlip(p=0.5),
                transforms.RandomRotation(degrees=45),
                transforms.ToTensor()
                ])
        
      self.test_transform = transforms.Compose([
                transforms.ToPILImage(),
                transforms.Resize((224, 224)),
                transforms.ToTensor()])
      
    def __len__(self):
      return len(self.df)
    
    def __getitem__(self,idx):

      row = self.df.loc[idx,:]
      pid = row['id']
      sub_df = self.df[self.df['id'] == pid]
        
      try: 
     
        img_id = np.random.choice(len(sub_df)) # randomly choose a patient image inside the fold 
        image = cv2.imread(str(self.path)  + '/' + self.folder + '/' + sub_df.iloc[img_id,:]['image_rename'])
        image = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
        img = self.train_transform(image) if self.is_train else self.test_transform(image)
         
        tab = torch.from_numpy(sub_df.iloc[img_id,:].drop(train_cols_to_drop).values.astype(np.float)).float() # dtype = torch.float32
        target = torch.tensor(row['OS'])
        return (img,tab), target , torch.tensor(sub_df.iloc[img_id]['id_encoding']) , torch.tensor(sub_df.iloc[img_id]['image_rename_encoding'])
        
      except Exception as e:
        print(e)
        print(pid, img_id)<jupyter_output><empty_output><jupyter_text>Collate Function<jupyter_code>def collate_fn(b):
    xs, ys, pids,img_rename = zip(*b)
    imgs, tabs = zip(*xs) # doule tuple 
    return (torch.stack(imgs).float(),torch.stack(tabs).float()),torch.stack(ys).float() , torch.stack(pids) , torch.stack(img_rename)<jupyter_output><empty_output><jupyter_text>Generating Splits<jupyter_code>from sklearn.model_selection import KFold

def get_split_idxs(n_folds=5):
    kv = KFold(n_splits=n_folds)
    splits = []
    for i,(train_idx, valid_idx) in enumerate(kv.split(unique_patient_ids)):
        splits.append((train_idx, valid_idx))
        
    return splits
splits = get_split_idxs(n_folds=config.FOLDS)
splits<jupyter_output><empty_output><jupyter_text>Training propsed modelGOAL:```Image Data --> CNNLayer                     \                     ---> FCLayer ---> Output                    /Metadata --> FCLayer```  Loading Supervised Pretrained Original paper: https://arxiv.org/pdf/2011.13971.pdfRepo: https://github.com/ozanciga/self-supervised-histopathology/tree/tenpercent<jupyter_code>RETURN_PREACTIVATION = True  # return features from the model, if false return classification logits
NUM_CLASSES = 1  # only used if RETURN_PREACTIVATION = False


def load_model_weights(model, weights):

    model_dict = model.state_dict()
    weights = {k: v for k, v in weights.items() if k in model_dict}
    if weights == {}:
        print('No weight could be loaded..')
    model_dict.update(weights)
    model.load_state_dict(model_dict)

    return model


model = torchvision.models.__dict__['resnet18'](pretrained=False)
state = torch.load(TEN_PERCENT_MODEL_PATH, map_location='cuda:0')

state_dict = state['state_dict']
for key in list(state_dict.keys()):
    state_dict[key.replace('model.', '').replace('resnet.', '')] = state_dict.pop(key)

model = load_model_weights(model, state_dict)

if RETURN_PREACTIVATION:
    model.fc = torch.nn.Sequential()
else:
    model.fc = torch.nn.Linear(model.fc.in_features, NUM_CLASSES)

model = model.cuda()

images = torch.rand((10, 3, 224, 224), device='cuda')

out = model(images)


for param in model.parameters():
    param.requires_grad = False
    # Replace the last fully-connected layer
    # Parameters of newly constructed modules have requires_grad=True by default
    
    model.fc = torch.nn.Linear(512, 256,bias = True) 
    model = model.cuda()

import copy

proposed_model = copy.deepcopy(model)

count = 1
for param in proposed_model.parameters():
  count += 1

cur = 1
for param in proposed_model.parameters():
  if (cur > (0.8 * count)): 
    param.requires_grad = True
  cur += 1

proposed_model = proposed_model.cuda()<jupyter_output><empty_output><jupyter_text>Training Global Model<jupyter_code>def train_loop(model, dl, opt , device, loss_fn):
    model.train()
    final_loss = []
    for X,y,_,_ in dl:
        imgs = X[0].to(device)
        tabs = X[1].to(device)
        y = y.to(device)
        outputs = model(imgs, tabs)
        loss = loss_fn(outputs.squeeze(), y)
        opt.zero_grad()
        loss.backward()
        opt.step()
        final_loss.append(loss.detach().cpu().numpy())
        #print('Training Example preds vs targets: ::\t \n' , outputs , '\n' , y)
    return np.array(final_loss).mean()
    
def eval_loop(model, dl, device, loss_fn):
    model.eval()
    final_outputs = []
    final_loss = []
    with torch.no_grad():
        for X,y,_,_ in dl:
            imgs = X[0].to(device)
            tabs = X[1].to(device)
            y=y.to(device)

            outputs = model(imgs, tabs)
            loss = loss_fn(outputs.squeeze(), y)

            final_outputs.extend(outputs.detach().cpu().numpy().tolist())
            final_loss.append(loss.detach().cpu().numpy())

    print('validation Example preds vs targets: ::\t \n' , outputs , '\n' , y)
    return final_outputs, final_loss
class Global_Model(nn.Module): # model to combina both tab and imag 
    def __init__(self , pretrained_medical_resnet):
        super().__init__()
        self.conv_model = pretrained_medical_resnet # after freezing some layers and setting regression node as 1 not 4 
        self.tab_model = nn.Sequential(nn.Linear(11, 500),
                                  nn.BatchNorm1d(500),
                                  nn.ReLU(),
                                  nn.Dropout(p=0.2),
                                  nn.Linear(500,250),
                                  nn.BatchNorm1d(250),
                                  nn.ReLU(),
                                  nn.Dropout(p=0.2))
                                  
        self.output =             nn.Sequential(
                                                nn.Linear(250 + 256 ,250),
                                                nn.ReLU(),
                                                nn.Dropout(p=0.3),
                                                nn.Linear(250, 125),
                                                nn.ReLU(),
                                                nn.Dropout(p=0.3),
                                                nn.Linear(125, 60),
                                                nn.ReLU(),
                                                nn.Dropout(p=0.3),
                                                nn.Linear(60, 20),
                                                nn.ReLU(),
                                                nn.Dropout(p=0.3),
                                                nn.Linear(20, 1, bias = True ))
    def forward(self, x,tab): # x image , tab is tabular data related 
        x = self.conv_model(x)
        tab_out = self.tab_model(tab)
        x = torch.cat([x, tab_out],dim=1)
        return self.output(x)
models = {}
for i in range(config.FOLDS):
    models[i] = copy.deepcopy(Global_Model(copy.deepcopy(proposed_model)))  
history = []
trainhistory = []

bestModels = []
bestScores = []

for i, (train_idx, valid_idx) in enumerate(splits):
    bestModel = None
    bestHistory = 100

    print(f"===================Fold : {i} ================")
    print("train_idx : " , train_idx , "\n valid_idx" , valid_idx)

    train = train_df.loc[train_df['id'].isin(unique_patient_ids[train_idx])].reset_index(drop=True)
    valid = train_df.loc[train_df['id'].isin(unique_patient_ids[valid_idx])].reset_index(drop=True)
    
    train_ds = Dataset(path, train,'Train' ,1)
    train_dl = torch.utils.data.DataLoader(dataset=train_ds,batch_size=config.TRAIN_BS,shuffle=True,collate_fn=collate_fn)

    valid_ds = Dataset(path, valid,'Train' ,0)
    valid_dl = torch.utils.data.DataLoader(dataset=valid_ds,batch_size=config.VALID_BS, shuffle=True,collate_fn=collate_fn)

    model = models[i]
    model.to(config.DEVICE)
    lr=1e-3
    momentum = 0.9
    
    num_steps = len(train_dl)
    optimizer = Adam(model.parameters(), lr=lr,weight_decay=0.01)

    train_losses = []
    losses = []
    for epoch in range(config.EPOCHS):
        print(f"=================EPOCHS {epoch+1}================")
        train_loss = train_loop(model, train_dl, optimizer, config.DEVICE,config.loss_fn)
        metrics = eval_loop(model, valid_dl, config.DEVICE,config.loss_fn)
        total_loss = np.array(metrics[1]).mean()

        if (total_loss < bestHistory):
          print('New best model! Score: ', total_loss)
          patienceCounter = 0
          bestHistory = total_loss
          bestModel = copy.deepcopy(model)
        
        if (epoch > patienceCounter):
          if (total_loss >  bestHistory):
            patienceCounter += 1
            print('BAD! Patience counter : ', patienceCounter)

        
        if (patienceCounter >= patience):
          print('Early stopping saved')
          break
        
        losses.append(total_loss)
        train_losses.append(train_loss)
        print('Train Loss:: \t', train_loss)
        print("Validation Loss ::\t", total_loss)
        
    model.to('cpu')

    history.append(losses)
    bestModels.append(bestModel)
    bestScores.append(bestHistory)<jupyter_output>===================Fold : 0 ================
train_idx :  [ 8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
 32 33 34 35 36 37] 
 valid_idx [0 1 2 3 4 5 6 7]
=================EPOCHS 1================<jupyter_text>Model utils like graphs and saving<jupyter_code>from matplotlib.pyplot import figure
import matplotlib.pyplot as plt

figure(figsize=(6, 8), dpi=80)

f, axes = plt.subplots(len(history), 1)
f.set_size_inches(12, 20)
for i in range(len(history)):
  axes[i].plot(history[i])
  #axes[i].plot(trainhistory[i])

  axes[i].legend(['Validation Loss'])
  axes[i].set_title(' Fold ' + str(i))
# Get best val loss scores per fold
for m in history:
  print(min(m))
# Save best models
c = 0
for m in bestModels:
    torch.save(m.state_dict(), OUTPUT_MODEL_PATH + f'/tab_best_fold_{c}.pth')
    c += 1
for k, m in models.items():
    torch.save(m.state_dict(), OUTPUT_MODEL_PATH + f'/fold_{k}.pth')<jupyter_output><empty_output><jupyter_text>Testing Pipeline<jupyter_code>fold_models_directory = Path(OUTPUT_MODEL_PATH)
fold_models_directory.ls()[6:11]
model_device = torch.device('cpu')
for i , fold_model_path in enumerate(fold_models_directory.ls()[6:11]) :
    models[i].load_state_dict(torch.load(str(fold_model_path) , map_location = model_device), strict=False)
test_df['OS'] = 0
test_pred_dict = {}
submissionDF = pd.DataFrame()

test_ds = Dataset(path, test_df,'Test' ,0)
test_dl = torch.utils.data.DataLoader(dataset=test_ds,batch_size=1, shuffle=False,collate_fn=collate_fn)
encoding_dic = {k:v for k,v in dict(tuple(test_df[['id' ,'id_encoding']].groupby(['id','id_encoding']))).keys()}
decoding_dic = {v:k for k,v in encoding_dic.items()} 
fold_count_dict = dict(zip(test_df['id'].unique().flatten(), np.zeros(9).flatten())) # count number of occurence for a cetain patient id in the batch 
fold_mean_dict = dict(zip(test_df['id'].unique().flatten(), np.zeros(9,dtype=int).flatten())) # compute mean fold 
final_submit_df = pd.DataFrame(columns = fold_mean_dict.keys()) 
# Test Set 
for key , fold_model in models.items() : 
  print(f"=================Predictions of fold Model {key}================")
  fold_model.to(config.DEVICE)
  fold_model.eval()
  with torch.no_grad():
      for X,y,pid,_ in test_dl:
          imgs = X[0].to(config.DEVICE)
          tabs = X[1].to(config.DEVICE)
          y=y.to(config.DEVICE)
          patient_id = pid.detach().cpu().numpy()[0]
          outputs = fold_model(imgs, tabs).detach().cpu().numpy()[0]

          fold_count_dict[decoding_dic[patient_id]]+=1
          fold_mean_dict[decoding_dic[patient_id]] += outputs
           
      fold_mean_dict = {k : v / fold_count_dict[k]  for k,v in fold_mean_dict.items()}
      print ("Fold count dict : " , fold_count_dict)
      print ("Fold mean dict after dividing : " , fold_mean_dict)
    
      final_submit_df.loc[len(final_submit_df)] = [v[0] for v in fold_mean_dict.values()]
      fold_mean_dict = dict(zip(test_df['id'].unique().flatten(), np.zeros(9,dtype=int).flatten())) # reset for next fold 
      fold_count_dict = dict(zip(test_df['id'].unique().flatten(), np.zeros(9).flatten()))

final_submit_df
final_submit_df.mean(axis=0)
final_submit_df.std(axis=0)
submissionDF = pd.DataFrame(final_submit_df.mean(axis=0)).reset_index().rename({'index' : 'id' , 0 : 'prediction' } , axis=1 )
submissionDF
submissionDF.to_csv("submission_V1.csv" , index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13305348852000254, 'fraction_numerical': 0.016218278954398015, 'mean_word_length': 2.8146530810286268, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 44, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '389191', 'license': 'MIT', 'n_tokens_mistral': 5361, 'n_tokens_neox': 5081, 'n_words': 1059, 'path': 'Run_Model_ipynb.ipynb', 'repo_name': 'BurgundyIsAPublicEnemy/ORLIAn', 'size': '15723'}