<jupyter_start><jupyter_text>Slice wise inference<jupyter_code>normal_path = config['inference']['dataloader']['normal_path']
abnormal_path = config['inference']['dataloader']['abnormal_path']
resize = tuple(config['inference']['dataloader']['resize'])
patch_size = tuple(config['inference']['dataloader']['patchsize'])
margin = tuple(config['inference']['dataloader']['margin'])
batch_size = config['inference']['dataloader']['batch']
num_workers = config['inference']['dataloader']['num_workers']
h_size = config['inference']['model']['h_size']
input_size = config['inference']['model']['input_size']
z_dim = config['inference']['model']['z_dim']
lamda = torch.tensor(config['inference']['model']['lamda'])
beta = torch.tensor(config['inference']['model']['beta'])
model_path = config['inference']['model']['load_model']
model_module = enc_dec.VAE(input_size, h_size, z_dim)
model = torch.load(model_path,map_location=device)
normal_data = cevae_loader.cevae(normal_path,patchsize=patch_size,margin=margin,resize=resize)
normal_loader = DataLoader(normal_data,batch_size=batch_size,num_workers=num_workers)
abnormal_data = cevae_loader.cevae(abnormal_path,patchsize=patch_size,margin=margin,resize=resize)
abnormal_loader = DataLoader(abnormal_data,batch_size=batch_size,num_workers=num_workers)
normal_score = []
model.eval()
with torch.no_grad():
    for idx,(data,masked_data) in enumerate(tqdm(normal_loader,desc='val_iter',leave=False)):
        data, masked_data = data.to(device), masked_data.to(device)
        rec_vae,mu, std = model(data)
        rec_ce,_,_ = model(masked_data)

        kl_loss = loss_fn.kl_divergence(mu,std)
        rec_loss_vae = loss_fn.rec_loss_fn(rec_vae,data)
        loss_vae = rec_loss_vae + kl_loss*beta
        rec_loss_ce = loss_fn.rec_loss_fn(rec_ce,data)
        loss = (1 - lamda)*loss_vae + lamda*rec_loss_ce
        normal_score.append(loss)
abnormal_score = []
model.eval()
with torch.no_grad():
    for idx,(data,masked_data) in enumerate(tqdm(abnormal_loader,desc='iter',leave=False)):
        data, masked_data = data.to(device), masked_data.to(device)
        rec_vae,mu, std = model(data)
        rec_ce,_,_ = model(masked_data)

        kl_loss = loss_fn.kl_divergence(mu,std)
        rec_loss_vae = loss_fn.rec_loss_fn(rec_vae,data)
        loss_vae = rec_loss_vae + kl_loss*beta
        rec_loss_ce = loss_fn.rec_loss_fn(rec_ce,data)
        loss = (1 - lamda)*loss_vae + lamda*rec_loss_ce
        abnormal_score.append(loss)
auroc,aupr,roc_curve,pr_curve = metrics.get_metrics(abnormal_score,normal_score)
auroc,aupr
fpr,tpr,_ = roc_curve
precision,recall, _ = pr_curve
def save_scores(array,filename,savepath):
    np.save(savepath+filename,array)
save_scores(fpr,'false_positive_slicewise_CEVAE','./results/')
save_scores(tpr,'True_positive_slicewise_CEVAE','./results/')
save_scores(precision,'precision_slicewise_CEVAE','./results/')
save_scores(recall,'recall_slicewise_CEVAE','./results/')<jupyter_output><empty_output><jupyter_text>Volume wise scores<jupyter_code>normal_volume = config['inference']['dataloader']['normal_volumes']
abnormal_volume = config['inference']['dataloader']['abnormal_volumes']
normal_vdata = cevae_loader.cevae(normal_volume,patchsize=patch_size,margin=margin,resize=resize)
abnormal_vdata = cevae_loader.cevae(abnormal_volume,patchsize=patch_size,margin=margin,resize=resize)
normal_vloader = DataLoader(normal_vdata,batch_size=batch_size,num_workers=num_workers)
abnormal_vloader = DataLoader(abnormal_vdata,batch_size=batch_size,num_workers=num_workers)
volume_normal = []
model.eval()
with torch.no_grad():
    for idx,(data,masked_data) in enumerate(tqdm(normal_vloader,desc='iter',leave=False)):
        data, masked_data = data.to(device), masked_data.to(device)
        rec_vae,mu, std = model(data)
        rec_ce,_,_ = model(masked_data)

        kl_loss = loss_fn.kl_divergence(mu,std)
        rec_loss_vae = loss_fn.rec_loss_fn(rec_vae,data)
        loss_vae = rec_loss_vae + kl_loss*beta
        rec_loss_ce = loss_fn.rec_loss_fn(rec_ce,data)
        loss = (1 - lamda)*loss_vae + lamda*rec_loss_ce
        volume_normal.append(loss)
volume_abnormal = []
model.eval()
with torch.no_grad():
    for idx,(data,masked_data) in enumerate(tqdm(abnormal_vloader,desc='iter',leave=False)):
        data, masked_data = data.to(device), masked_data.to(device)
        rec_vae,mu, std = model(data)
        rec_ce,_,_ = model(masked_data)

        kl_loss = loss_fn.kl_divergence(mu,std)
        rec_loss_vae = loss_fn.rec_loss_fn(rec_vae,data)
        loss_vae = rec_loss_vae + kl_loss*beta
        rec_loss_ce = loss_fn.rec_loss_fn(rec_ce,data)
        loss = (1 - lamda)*loss_vae + lamda*rec_loss_ce
        volume_abnormal.append(loss)
vauroc,vaupr,vroc, vpr = metrics.get_metrics(volume_abnormal,volume_normal)
vauroc,vaupr
vfpr,vtpr,_ = vroc
vprec,vrec,_ = vpr
save_scores(vfpr,'false_positive_clinicwise_CEVAE','./results/')
save_scores(vtpr,'True_positive_clinicwise_CEVAE','./results/')
save_scores(vprec,'precision_clincwise_CEVAE','./results/')
save_scores(vrec,'recall_clinicwise_CEVAE','./results/')<jupyter_output><empty_output><jupyter_text>Mask creation and segmentation evaluation<jupyter_code>from utils.image_utils import *
from utils.mask_utils import *
import matplotlib.pyplot as plt
data = load_data(abnormal_path,resize)
dataloader = DataLoader(data,batch_size=batch_size,num_workers=num_workers)
save_path = './masks/cevae_masks/'
with torch.no_grad():
    for idx,(data,filename) in enumerate(dataloader):
        data = data.float()
        data = data.to(device)
        output,mu,std = model(data)
        mask = output - data
        mask = mask.detach().cpu().numpy()
        mask = np.squeeze(mask,axis=0)
        mask = normalise_mask(mask[:,:,:])
        mask = mask < 0.45
        save_masks(mask,filename[0],save_path)<jupyter_output><empty_output><jupyter_text>segmentation metric<jupyter_code>gt_path = './masks/segmentations/'
mask_path = './masks/cevae_masks/'
gt_files = sorted(glob(gt_path+'*.npy',recursive=True))
mask_files = sorted(glob(mask_path+'*.npy',recursive=True))
len(gt_files),len(mask_files)
dice_scores = []
for i in tqdm(range(len(mask_files))):
    mask_file = np.load(mask_files[i])
    gt_file = np.load(gt_files[i])
    dice_scores.append(metrics.dice(gt_file,mask_file))
avg_dice = np.array(dice_scores).mean()
avg_dice<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1405152224824356, 'fraction_numerical': 0.001405152224824356, 'mean_word_length': 6.263038548752834, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '538654', 'license': 'MIT', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2323, 'n_words': 271, 'path': 'Inference.ipynb', 'repo_name': 'AbinavRavi/CeVAE', 'size': '6405'}