<jupyter_start><jupyter_text>Preparing libraries<jupyter_code># libraries
import os

os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES']='4,5,6,7'                       ### specify GPU number here

# torch.cuda.set_device(0)   
# torch.cuda.set_device(1)   
# torch.cuda.set_device(2)   
# torch.cuda.set_device(3)   

import matplotlib.pyplot as plt
from matplotlib.image import imread

import numpy as np
import pandas as pd

import torch
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")


from torch.utils.data import TensorDataset, DataLoader,Dataset
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
from torch.optim import lr_scheduler

from sklearn import svm, datasets
from sklearn.utils.multiclass import unique_labels

from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold,StratifiedKFold
from sklearn.metrics import roc_auc_score

import seaborn as sns

import torch.backends.cudnn as cudnn
import time 
import tqdm
import random
from PIL import Image
train_on_gpu = True
from torch.utils.data.sampler import SubsetRandomSampler
from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR
from utils import print_confusion_matrix
import cv2 
from sklearn.utils import shuffle


import albumentations
from albumentations import torch as AT
#import pretrainedmodels

import scipy.special

from pytorchcv.model_provider import get_model as ptcv_get_model

cudnn.benchmark = True
import warnings
warnings.filterwarnings("ignore")
from torch.utils import data
from PIL import Image

from processing_pytorch import CancerDataset, generate_dataset_tissue_type, df_dl_features<jupyter_output><empty_output><jupyter_text>define constant<jupyter_code>INPUT_SHAPE = 224
SEED = 323
def seed_everything(seed=SEED):
    random.seed(seed)
    os.environ['PYHTONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
seed_everything(SEED)
main_path = '.\\'<jupyter_output><empty_output><jupyter_text>Preparing data<jupyter_code>X_train, y_train,train_paths = generate_dataset_tissue_type(main_path,os.path.join(main_path,'data_train_stroma_vs_epithelial_tissue.csv'),SEED)
X_test, y_test,test_paths = generate_dataset_tissue_type(main_path,os.path.join(main_path,'data_test_stroma_vs_epithelial_tissue.csv'),SEED)
X_val, y_val,val_paths = generate_dataset_tissue_type(main_path,os.path.join(main_path,'data_valid_stroma_vs_epithelial_tissue.csv'),SEED)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>data_transforms = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.RandomRotate90(p=0.5),
    albumentations.Transpose(p=0.5),
    albumentations.Flip(p=0.5),
    albumentations.OneOf([
        albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen(), albumentations.IAAEmboss(), 
        albumentations.RandomBrightness(), albumentations.RandomContrast(),
        albumentations.JpegCompression(), albumentations.Blur(), albumentations.GaussNoise()], p=0.5), 
    albumentations.HueSaturationValue(p=0.5), 
    albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

data_transforms_test = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

data_transforms_tta0 = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.RandomRotate90(p=0.5),
    albumentations.Transpose(p=0.5),
    albumentations.Flip(p=0.5),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

data_transforms_tta1 = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.RandomRotate90(p=1),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

data_transforms_tta2 = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.Transpose(p=1),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

data_transforms_tta3 = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.Flip(p=1),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])

dataset = CancerDataset(X_train, y_train,  transform=data_transforms)
test_set = CancerDataset(X_test, y_test,  transform=data_transforms_test)
val_set = CancerDataset(X_val, y_val,  transform=data_transforms_test)

batch_size = 16
num_workers = 0
# # prepare data loaders (combine dataset and sampler)
train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=None, num_workers=num_workers)
valid_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, sampler=None, num_workers=num_workers)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>model_conv = ptcv_get_model("cbam_resnet50", pretrained=True)
model_conv.avg_pool = nn.AdaptiveAvgPool2d((1, 1)).cuda()
model_conv.last_linear = nn.Sequential(nn.Dropout(0.6), nn.Linear(in_features=2048, out_features=512, bias=True), nn.SELU(),
                                      nn.Dropout(0.8),  nn.Linear(in_features=512, out_features=1, bias=True)).cuda()
model_conv.cuda()
criterion = nn.BCEWithLogitsLoss() #binary cross entropy with sigmoid

optimizer = optim.Adam(model_conv.parameters(), lr=0.0004)

scheduler = StepLR(optimizer, 5, gamma=0.2)
scheduler.step()
if torch.cuda.device_count() > 1:
    print(torch.cuda.device_count() )
    model_conv = nn.DataParallel(model_conv,device_ids=[0,1,2,3])
model_conv.to(device)
val_auc_max = 0
patience = 5
# current number of tests, where validation loss didn't increase
p = 0
# whether training should be stopped
stop = False

# number of epochs to train the model
n_epochs = 20
for epoch in range(1, n_epochs+1):
    
    if stop:
        print("Training stop.")
        break
        
    print(time.ctime(), 'Epoch:', epoch)

    train_loss = []
    train_auc = []
        
    for tr_batch_i, (data, target) in enumerate(train_loader):
        
        model_conv.train()

        data, target = data.cuda(), target.cuda()
        #data, target = data.to(device), target.to(device)

        optimizer.zero_grad()
        output = model_conv(data)
        loss = criterion(output[:,0], target.float())
        train_loss.append(loss.item())
        
        a = target.data.cpu().numpy()
        try:
            b = output[:,0].detach().cpu().numpy()
            train_auc.append(roc_auc_score(a, b))
        except:
            pass

        loss.backward()
        optimizer.step()
        
        if (tr_batch_i+1)%600 == 0:  
            #model_conv = nn.DataParallel(model_conv)
            model_conv.eval()
            val_loss = []
            val_auc = []
            for val_batch_i, (data, target) in enumerate(valid_loader):
                data, target = data.cuda(), target.cuda()
                #data, target = data.to(device), target.to(device)
                output = model_conv(data)

                loss = criterion(output[:,0], target.float())

                val_loss.append(loss.item()) 
                a = target.data.cpu().numpy()
                try:
                    b = output[:,0].detach().cpu().numpy()
                    val_auc.append(roc_auc_score(a, b))
                except:
                    pass

            print('Epoch %d, batches:%d, train loss: %.4f, valid loss: %.4f.'%(epoch, tr_batch_i, np.mean(train_loss), np.mean(val_loss)) 
                  + '  train auc: %.4f, valid auc: %.4f'%(np.mean(train_auc),np.mean(val_auc)))
            train_loss = []
            train_auc = []
            valid_auc = np.mean(val_auc)
            if valid_auc > val_auc_max:
                print('Validation auc increased ({:.6f} --> {:.6f}).  Saving model ...'.format(
                val_auc_max,
                valid_auc))
                torch.save(model_conv.state_dict(), r".//model_epoch_{}_val_{:.4f}.pt".format(epoch ,(valid_auc*1000)))
                #torch.save(model_conv.state_dict(), 'model.pt')
                val_auc_max = valid_auc
                p = 0
            else:
                p += 1
                if p > patience:
                    print('Early stop training')
                    stop = True
                    break   
            scheduler.step()
torch.cuda.empty_cache()<jupyter_output><empty_output><jupyter_text>Generate deep learning features<jupyter_code>model_conv.eval()
saved_dict = torch.load(r".\\model_epoch_2_val_952.8703_with_valid_test.pt")

model_conv.load_state_dict(saved_dict)

new_classifier = nn.Sequential(*list(model_conv.children())[-1].features).cpu()

INPUT_SHAPE = 224
data_transforms = albumentations.Compose([
    albumentations.Resize(INPUT_SHAPE, INPUT_SHAPE),
    albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
    AT.ToTensor()
    ])
train_features = df_dl_features(X_train,train_paths,data_transforms,new_classifier)
train_features.to_csv(r".\train_features_tissue_type_he.csv")
test_features = df_dl_features(X_test,test_paths,data_transforms,new_classifier)
test_features.to_csv(r".\test_features_tissue_type_he.csv")
valid_features = df_dl_features(X_val,val_paths,data_transforms,new_classifier)
valid_features.to_csv(r".\valid_features_tissue_type_he.csv")<jupyter_output><empty_output><jupyter_text>TTA inference<jupyter_code>NUM_TTA = 10
sigmoid = lambda x: scipy.special.expit(x)
def def_tta(X_data,y_data):
    for num_tta in range(NUM_TTA):
        if num_tta==0:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms_test)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
        elif num_tta==1:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms_tta1)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
        elif num_tta==2:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms_tta2)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
        elif num_tta==3:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms_tta3)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
        elif num_tta<8:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms_tta0)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
        else:
            test_set = CancerDataset(X_data, y_data,  transform=data_transforms)
            test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)

        preds = []
        for batch_i, (data, target) in enumerate(test_loader):
            data, target = data.cuda(), target.cuda()
            output = model_conv(data).detach()
            pr = output[:,0].cpu().numpy()
            for i in pr:
                preds.append(sigmoid(i)/NUM_TTA)
        if num_tta==0:
            test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
            test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
        else:
            test_preds['preds']+=np.array(preds)
        print(num_tta)
    return(test_preds)

test_preds = def_tta(X_test,y_test)
valid_preds = def_tta(X_val,y_val)<jupyter_output><empty_output><jupyter_text>generate figures<jupyter_code>predictions_test = np.array(test_preds['preds'] > 0.5).astype(int)
validations_test=np.array(valid_preds['preds'] > 0.5).astype(int)
new_y_test = []
for elmt in y_test:
    if elmt == 0:
        new_y_test.append("stroma")
    elif elmt == 1:
        new_y_test.append("epithelial tissue")

new_y_valid = []
for elmt in y_val:
    if elmt == 0:
        new_y_valid.append("stroma")
    elif elmt == 1:
        new_y_valid.append("epithelial tissue")
        
new_pred_test = []
for elmt in predictions_test:
    if elmt == 0:
        new_pred_test.append("stroma")
    elif elmt == 1:
        new_pred_test.append("epithelial tissue")

new_pred_valid = []
for elmt in validations_test:
    if elmt == 0:
        new_pred_valid.append("stroma")
    elif elmt == 1:
        new_pred_valid.append("epithelial tissue")

fig =print_confusion_matrix(np.array(new_y_valid),new_pred_valid , class_names=["stroma","epithelial tissue"],normalize=True)
fig.savefig(r".\new_msi_norm_tissue_validation")
fig_2 = print_confusion_matrix(np.array(new_y_test),new_pred_test, class_names=["stroma","epithelial tissue"],normalize=True)
fig_2.savefig(r".\new_2_msi_norm_tissue_test")

fpr_test, tpr_test, thresholds_test = sklearn.metrics.roc_curve(np.array(y_test),np.array(test_preds))
roc_auc_test = sklearn.metrics.auc(fpr_test, tpr_test)
fpr_val, tpr_val, thresholds_val = sklearn.metrics.roc_curve(np.array(y_val),np.array(valid_preds))
roc_auc_val = sklearn.metrics.auc(fpr_val, tpr_val)
plt.figure()
lw = 2
plt.plot(fpr_test, tpr_test,
         label='ROC curve test dataset (area = {0:0.2f})'
               ''.format(roc_auc_test),
         color='darkorange', linewidth=2)

plt.plot(fpr_val, tpr_val,
         label='ROC curve validation dataset (area = {0:0.2f})'
               ''.format(roc_auc_val),
         color='green',  linewidth=2)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic tissue type')
plt.legend(loc="lower right")
plt.savefig(r'.\new_roc_curves_valid_test.png', bbox_inches='tight')
plt.show()
report = sklearn.metrics.classification_report(np.array(new_y_valid), np.array(new_pred_valid), output_dict=True)
df = pd.DataFrame(report).transpose()
df.to_excel(r".\classification_report_HE_gland_vs_tissue_valid.xlsx",index=None)

report = sklearn.metrics.classification_report(np.array(new_y_test), np.array(new_pred_test), output_dict=True)
df = pd.DataFrame(report).transpose()
df.to_excel(r".\classification_report_HE_gland_vs_tissue_test.xlsx",index=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12351785351311065, 'fraction_numerical': 0.02384985432617386, 'mean_word_length': 3.900398406374502, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '250221', 'license': 'Apache-2.0', 'n_tokens_mistral': 5526, 'n_tokens_neox': 5241, 'n_words': 808, 'path': 'pytorch_HE_stroma_vs_epithelial_tissue.ipynb', 'repo_name': 'precision-medicine-um/ML-and-MS-in-esophageal-cancer', 'size': '14759'}