<jupyter_start><jupyter_text>AI61002_Spr2020 Project 2: COVID detection using CXR<jupyter_code>%matplotlib inline
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from torchvision import transforms, datasets
import torchvision.transforms.functional as TF
from torch.utils.data import DataLoader, Dataset
import torch.optim as optim
import torchvision.models as models
import pandas as pd
from PIL import Image, ImageOps
from skimage import io, util, transform as tf
from skimage.io._plugins.pil_plugin import pil_to_ndarray, ndarray_to_pil
import os, shutil
import random
import albumentations
import cv2
import math
import seaborn as sns
from PIL import Image
from collections import Counter
from sklearn.utils import resample
from skimage import io, transform
import time
import copy
# Mouting to google drive where all the data was downloaded
from google.colab import drive
drive.mount('/content/gdrive')
os.mkdir('Project_2')
os.chdir('Project_2')
os.mkdir('CXR_files')
os.chdir('/content/gdrive/My Drive/COVID_db/Project_2')
! ls -ahtr<jupyter_output>Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount("/content/gdrive", force_remount=True).
CXR_files	    stage_2_test_images   dummy.jpeg
Train_Combined.csv  stage_2_train_images  15EC34002_Project2_NEW.ipynb
Test_Combined.csv   CheXpert-v1.0<jupyter_text>Downloading data from different sources using standard linux commands<jupyter_code>! wget "https://ieee-dataport.s3.amazonaws.com/open/24957/Covid19action-radiology-CXR_v1.1.zip?response-content-disposition=attachment%3B%20filename%3D%22Covid19action-radiology-CXR_v1.1.zip%22&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAJOHYI4KJCE6Q7MIQ%2F20200516%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200516T102638Z&X-Amz-SignedHeaders=Host&X-Amz-Expires=86400&X-Amz-Signature=2de09bd7599e24288c647f256acd845f826b27ac191ca7f1e770787c44cc5b01" -O ieee_dp.zip
! unzip "ieee_dp.zip" > /dev/null
! rm -f ieee_dp.zip
! mv Covid19action-radiology-CXR_v1.1/*.csv .
! mv Covid19action-radiology-CXR_v1.1/images/Source1/* CXR_files/
! mv Covid19action-radiology-CXR_v1.1/images/Source2/* CXR_files/
! mv Covid19action-radiology-CXR_v1.1/images/Source7/* CXR_files/
! rm -rf Covid19action-radiology-CXR_v1.1

! wget http://download.cs.stanford.edu/deep/CheXpert-v1.0-small.zip
! unzip CheXpert-v1.0-small.zip > /dev/null && rm -rf CheXpert-v1.0-small.zip
! mkdir -p CXR_files/Train_reshape/
! mkdir -p CXR_files/Valid_reshape/

  arrange_cheXpert_Data()

! wget "https://storage.googleapis.com/kaggle-competitions-data/kaggle-v2/10338/862042/bundle/archive.zip?GoogleAccessId=<EMAIL>&Expires=1589885316&Signature=SDguo%2FG3u9Rldz0kshG6lDUHWo%2BsTeoNteah4Gec8Q7bfn7NkB%2Fm8en8RpwumtJS%2Bz%2FCXfoNg5b5Xu2Who%2FXm65ltxCQMS2nzTK7I%2BQZGJGFGk3dHDHlKxF0eZXyL4CPNQpn7sqCXDXck0RcZ%2FKsfagfuP96mtY6j7dhpn3un6DPyU9thuhj7JQj%2Fzkq8o2%2BpSMCzgE5fTa8wywIfsLPYqENpbWgBlxREoyAsRU9jRUTSzMtLRdwiipQvk4IaIl99TbV3iHiMXqhIAZXycalXEvkVnJpvvbJog4IP9AmvQz9vbQqAW24as%2FXaeieFwZYrn%2F5Q0zb37KFyvDKfXj3Yg%3D%3D&response-content-disposition=attachment%3B+filename%3Drsna-pneumonia-detection-challenge.zip" -O Source4.zip
! unzip Source4.zip -d src4_temp > /dev/null && rm -rf Source4.zip
  convert_dcm_to_jpg()
! mv src4_temp/stage_2_train_images/*jpg CXR_files/
! mv src4_temp/stage_2_test_images/*jpg CXR_files/
! rm -rf src4_temp

! git clone https://github.com/ieee8023/covid-chestxray-dataset
! mv covid-chestxray-dataset/images/* CXR_files/
! rm -rf covid-chestxray-dataset

! git clone https://github.com/agchung/Figure1-COVID-chestxray-dataset
! mv Figure1-COVID-chestxray-dataset/images/* CXR_files/
! rm -rf Figure1-COVID-chestxray-dataset

! ls -lahtr .

# Output is not shown because these commands were run only once in the beginning
def arrange_cheXpert_Data():
    root_dir = 'CheXpert-v1.0'
    for dir_, _, files in os.walk(root_dir):
        for file_name in files:
            rel_dir = os.path.relpath(dir_, root_dir)
            rel_file = os.path.join(root_dir, rel_dir, file_name)
            rel_file_name = rel_file.replace('/','__')
            if 'train' in rel_file_name:
                shutil.move(rel_file, f'CXR_files/Train_reshape/{rel_file_name}')
            else:
                shutil.move(rel_file, f'CXR_files/Valid_reshape/{rel_file_name}')

df = pd.read_csv("Train_Combined.csv")
print(df.describe())
print(df)
df["Data Source"].value_counts()
# Here are the counts of images from different sources<jupyter_output><empty_output><jupyter_text>Visualising the images by plotting some imgaes<jupyter_code>tmp = plot_images(1, "EUNQ4TDXkAAxvth.jfif", plot=True)
tmp = plot_images(2, "CASE58_XRAY_1.jpeg", plot=True)
tmp = plot_images(3, "Train_reshape/CheXpert-v1.0__train__patient39740__study3__view1_frontal.jpg", plot=True)
def plot_freq_chart(freq):
    sns.barplot(freq.index, freq.values)
    plt.title('Class frequency')
    plt.ylabel('Number of Occurrences', fontsize=12)
    plt.xlabel('city', fontsize=1)
    plt.show()<jupyter_output><empty_output><jupyter_text>Splitting between training  and validation sets Method below is based on https://stackoverflow.com/questions/3825010/how-to-split-data-into-3-n-validation-and-test<jupyter_code># 75, 25 split
train, validate = np.split(df.sample(frac=1), [int(.75*len(df))])<jupyter_output><empty_output><jupyter_text>Making custom dataloader<jupyter_code>class CXR_Dataset(Dataset):
    
    def __init__(self, df, is_test, col_no, transform=None):
        self.data = df
        self.transform = transform
        self.test = is_test
        self.col_no = col_no
        
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self, index):
        img_name = self.data.iloc[index, 0]
        source = int(self.data.iloc[index, 1].split("-")[-1])
        image = mod(source, img_name)
        label =self.data.iloc[index][3:][self.col_no]
        
        if self.transform is not None:
            # converting image to black and white
#             image = image.convert('1')
            #converting to rgb
            image = image.convert('RGB')
            # applying transforms
            image = self.transform(image)
        
        # for train loader it returns label and image both but for test loader returns only image 
        if self.test:
            return image
        else:
            return image, label
#The template has been adapted from https://github.com/iitkliv/ai61002_Spr2020/blob/master/Tutorial4_Training_LeNet_using_GD.ipynb and https://www.kaggle.com/c/digit-recognizer

def train_net(dataloaders, dataset_sizes, model, criterion, optimizer, scheduler, num_epochs=25):
    t_ini = time.time()
    use_gpu = True
    best_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    print(dataset_sizes)
    best_validation_loss = 1e9 # initializing with very high error

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()
                # save latest for hot restart
                states = {
                    'epoch': num_epochs,
                    'state_dict': model_ft.state_dict(),
                    'optimizer': optimizer.state_dict()
                }
                torch.save(states, "./Latest_{col}")

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_wts = copy.deepcopy(model.state_dict())
                states = {
                    'epoch': num_epochs,
                    'state_dict': model_ft.state_dict(),
                    'optimizer': optimizer_ft.state_dict(),
                    'lr_scheduler': exp_lr_scheduler.state_dict()
                }
                torch.save(states, f"./{col}_{BatchSize}_SGD_{lr}_{epoch}")
                
            # stopping criteria
            curr_val_loss = 0.0
            for i, data in enumerate(dataloaders['val']):
                inputs, labels = data
                if use_gpu:
                    inputs, labels = inputs.cuda(), labels.cuda()
                outputs = model_ft(inputs)
                preds_ = torch.argmax(outputs, dim=1)
                loss_ = criterion(outputs, labels)
                curr_val_loss += loss_

            if curr_val_loss < best_validation_loss:
                #best model
                states = {
                    'epoch': num_epochs,
                    'state_dict': model_ft.state_dict(),
                    'optimizer': optimizer.state_dict(),
                }
                torch.save(states, "./Best_LeNet_0_08.mdl")
                best_validation_loss = curr_val_loss
            else:
                #early stopping

                break

        print()

    t_el = time.time() - t_ini
    print('Training complete in {:.0f}m {:.0f}s'.format(
        t_el // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_wts)
    return model
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
# Here the size of each output sample is set to 2.
# Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
model_ft.fc = nn.Linear(num_ftrs, 3)

model_ft = model_ft.to(device)

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
print(model_ft)<jupyter_output>ResNet(
  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu): ReLU(inplace=True)
  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (layer1): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
  [...]<jupyter_text>Upsampled Non-Pneumonia<jupyter_code>def get_balanced(df, col):
    frequency_dict = train[col].value_counts()
    max_freq = frequency_dict[frequency_dict.index[0]] # 1 as counter indexes in decreasing order
    store_all = []
    for i in range(frequency_dict.shape[0]):
        temp_df = train[train[col] == i]
        temp_df_upsampled = resample(temp_df, replace=True, n_samples=max_freq, random_state=42)
        store_all.append(temp_df_upsampled)

    upsampled_df = pd.concat(store_all)
    return upsampled_df
col = "Non-Pneumonia"
ups_train = get_balanced(train, col)

# just checking if dataframe is oversampled correctly and is now balanced or not
up_val_cnts = ups_train[col].value_counts()
sns.barplot(up_val_cnts.index, upsampled_value_counts.values)
plt.title('Class frequency')
plt.ylabel('Number of Occurrences in train set', fontsize=12)
plt.xlabel('city', fontsize=1)
plt.show()

apply_transform = transforms.Compose([
    transforms.Resize((32, 32)), 
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
BatchSize = 512

train_ = CXR_Dataset(ups_train, False, 0, transform=apply_transform)
trainLoader = torch.utils.data.DataLoader(train_, batch_size=BatchSize,
                                          shuffle=True, num_workers=0) # Creating dataloader

validate_ = CXR_Dataset(validate, False, 0, transform=apply_transform)
validateLoader = torch.utils.data.DataLoader(validate_, batch_size=BatchSize,
                                          shuffle=False, num_workers=0)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features
# Here the size of each output sample is set to 2.
# Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
model_ft.fc = nn.Linear(num_ftrs, 3)

model_ft = model_ft.to(device)

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
lr = 0.001
num_epochs=25
# Learning rate and epocs chosen based on hyperparameters tuning
def final(train, validate, num_of_images):
    lr = 0.001
    num_epochs=25
    use_gpu = True
    
    validate = validate[:num_of_images]
    
    cols = ["Non-Pneumonia", "Other Pneumonia", "COVID-19"]
    for col in cols:
        ups_train = get_balanced(train, col)
        ups_train = upsampled_train[:num_of_images]
        
        apply_transform = transforms.Compose([
            transforms.Resize((32, 32)), 
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])
        BatchSize = 256
        print("Training df size ", ups_train.shape[0])
        train_ = CovidDataset(ups_train, False, 0, transform=apply_transform)
        trainLoader = torch.utils.data.DataLoader(train_, batch_size=BatchSize,
                                                  shuffle=True, num_workers=0) # Creating dataloader

        validate_ = CovidDataset(validate, False, 0, transform=apply_transform)
        validateLoader = torch.utils.data.DataLoader(validate_, batch_size=BatchSize,
                                          shuffle=False, num_workers=0)
        
        dataloaders = {
            'train': trainLoader,
            'val': validateLoader
        }

        dataset_sizes = {
            'train': len(train),
            'val': len(validate)
        }
        

        model_ft = models.resnet18(pretrained=True)
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs, 3)
        if use_gpu:
            model_ft = model_ft.cuda()

        criterion = nn.CrossEntropyLoss()

        # Observe that all parameters are being optimized
        optimizer_ft = optim.SGD(model_ft.parameters(), lr=lr, momentum=0.9)

        # Decay LR by a factor of 0.1 every 7 epochs
        exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
        
        model_ft = train_net(dataloaders, dataset_sizes, model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=num_epochs)

#Training the network
final(train, validate, 3000)<jupyter_output>Training df size  3000
{'train': 178964, 'val': 3000}
Epoch 0/24
----------
train Loss: 0.0112 Acc: 0.0118
val Loss: 1.6322 Acc: 0.0913

Epoch 1/24
----------
train Loss: 0.0004 Acc: 0.0168
val Loss: 2.6339 Acc: 0.0913

Epoch 2/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 4.1582 Acc: 0.0913

Epoch 3/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 5.8819 Acc: 0.0913

Epoch 4/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 7.3263 Acc: 0.0913

Epoch 5/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 8.4390 Acc: 0.0913

Epoch 6/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 9.3156 Acc: 0.0913

Epoch 7/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 9.8279 Acc: 0.0913

Epoch 8/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 10.2029 Acc: 0.0913

Epoch 9/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 10.4208 Acc: 0.0913

Epoch 10/24
----------
train Loss: 0.0000 Acc: 0.0168
val Loss: 10.5534 Acc: 0.0913

Epoch 11/24
----------
train [...]<jupyter_text>Testing accuracy<jupyter_code>test = pd.read_csv("Test_Combined.csv")
test_ = CXR_Dataset(test, False, 0, transform=apply_transform)
testLoader = torch.utils.data.DataLoader(test_, batch_size=BatchSize,
                                          shuffle=True, num_workers=0) # Creating dataloader

cols = ["Non-Pneumonia", "Other Pneumonia", "COVID-19"]
name = {
    cols[0]: 5,
    cols[1]: 3,
    cols[2]: 3
}
for col in cols:
    correct_pred = 0
    for data in testLoader:
        inputs,labels = data
        inputs, labels = inputs.cuda(),labels.cuda()
        #load model
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

        model_ft = models.resnet18(pretrained=True)
        num_ftrs = model_ft.fc.in_features
        # Here the size of each output sample is set to 2.
        # Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
        model_ft.fc = nn.Linear(num_ftrs, 3)

        model_ft = model_ft.to(device)
        checkpoint = torch.load(f"./{col}_{BatchSize}_SGD_{lr}_{name[col]}")
        model_ft.load_state_dict(checkpoint['state_dict'], strict=False)
        
        # Feedforward train data batch through model
        output = model_ft(inputs) 
        # Predicted class is the one with maximum probability
        preds = torch.argmax(output,dim=1)
        correct_pred += torch.sum(preds==labels)


    test_accuracy = correct_pred.item()/len(test)
    print('Testing accuracy = ',test_accuracy*100)<jupyter_output>Testing accuracy =  10.76923076923077
Testing accuracy =  0.3076923076923077
Testing accuracy =  0.0
{'fraction_non_alphanumeric': 0.11540854417254251, 'fraction_numerical': 0.04697221070095396, 'mean_word_length': 2.8936213161081956, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 30, 'https://': 7, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '464121', 'license': 'MIT', 'n_tokens_mistral': 7261, 'n_tokens_neox': 6603, 'n_words': 1387, 'path': '15EC34002_Project2.ipynb', 'repo_name': 'meta-ks/COVID19_assignment', 'size': '19327'}