<jupyter_start><jupyter_text>Detecting COVID-19 with Chest X Ray using PyTorchImage classification of Chest X Rays in one of three classes: Normal, Viral Pneumonia, COVID-19Notebook created for the guided project Detecting COVID-19 with Chest X Ray using PyTorch on CourseraDataset from COVID-19 Radiography Dataset on Kaggle  Import Libraries<jupyter_code># %matplotlib inline

import os
import shutil
import random
import torch
import torchvision
import numpy as np

from PIL import Image
from matplotlib import pyplot as plt

torch.manual_seed(0)

print('Using PyTorch version', torch.__version__)<jupyter_output><empty_output><jupyter_text>Preparing Training and Test Sets<jupyter_code>class_names = ['normal', 'viral', 'covid']
root_dir = '../data/COVID-19 Radiography Database'
source_dirs = ['Normal', 'Viral Pneumonia', 'COVID']

if os.path.isdir(os.path.join(root_dir, source_dirs[1])):
    os.mkdir(os.path.join(root_dir, 'test'))

    for i, d in enumerate(source_dirs):
        os.rename(os.path.join(root_dir, d), os.path.join(root_dir, class_names[i]))

    for c in class_names:
        os.mkdir(os.path.join(root_dir, 'test', c))

    for c in class_names:
        images = [x for x in os.listdir(os.path.join(root_dir, c)) if x.lower().endswith('png')]
        selected_images = random.sample(images, 30)
        for image in selected_images:
            source_path = os.path.join(root_dir, c, image)
            target_path = os.path.join(root_dir, 'test', c, image)
            shutil.move(source_path, target_path)<jupyter_output><empty_output><jupyter_text>Creating custom Dataset<jupyter_code>class ChestXRayDataset(torch.utils.data.Dataset):
    def __init__(self, image_dirs, transform):
        def get_images(class_name):
            images = [x for x in os.listdir(image_dirs[class_name]) if x.lower().endswith('png')]
            print(f'Found {len(images)} {class_name} examples')
            return images
        self.images = {}
        self.class_names = ['normal', 'viral', 'covid']
        
        for c in self.class_names:
            self.images[c] = get_images(c)
        
        self.image_dirs = image_dirs
        self.transform = transform

    def __len__(self):
        return sum([len(self.images[c]) for c in self.class_names])
        
    def __getitem__(self, index):
        class_name = random.choice(self.class_names)
        index = index % len(self.images[class_name])
        image_name = self.images[class_name][index]
        image_path = os.path.join(self.image_dirs[class_name],image_name)
        image = Image.open(image_path).convert('RGB')
        return self.transform(image), self.class_names.index(class_name)<jupyter_output><empty_output><jupyter_text>Image Transformations<jupyter_code>train_transform = torchvision.transforms.Compose([
    torchvision.transforms.Resize(size=(224, 224)),
    torchvision.transforms.RandomHorizontalFlip(),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

test_transform = torchvision.transforms.Compose([
    torchvision.transforms.Resize(size=(224, 224)),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])<jupyter_output><empty_output><jupyter_text>PrepareData Loaders<jupyter_code>print("Train Dataset")
train_dirs = {
    'normal': '../data/COVID-19 Radiography Database/normal',
    'viral': '../data/COVID-19 Radiography Database/viral',
    'covid': '../data/COVID-19 Radiography Database/covid'
}

train_dataset = ChestXRayDataset(train_dirs, train_transform)

print("\nTest Dataset")
test_dirs = {
    'normal': '../data/COVID-19 Radiography Database/test/normal',
    'viral': '../data/COVID-19 Radiography Database/test/viral',
    'covid': '../data/COVID-19 Radiography Database/test/covid'
}
test_dataset = ChestXRayDataset(test_dirs, test_transform)
batch_size = 6
train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size = batch_size, shuffle = True)
test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size = batch_size, shuffle = False)
print("Number of training batches ",len(train_data_loader))
print("Number of test batches ",len(test_data_loader))<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>class_names = train_dataset.class_names


def show_images(images, labels, preds):
    plt.figure(figsize=(8, 4))
    for i, image in enumerate(images):
        plt.subplot(1, 6, i + 1, xticks=[], yticks=[])
        image = image.numpy().transpose((1, 2, 0))
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])
        image = image * std + mean
        image = np.clip(image, 0., 1.)
        plt.imshow(image)
        col = 'green' if preds[i] == labels[i] else 'red'
            
        plt.xlabel(f'{class_names[int(labels[i].numpy())]}')
        plt.ylabel(f'{class_names[int(preds[i].numpy())]}', color=col)
    plt.tight_layout()
    plt.show()
images, labels = next(iter(train_data_loader))
show_images(images, labels, labels)
images, labels = next(iter(test_data_loader))
show_images(images, labels, labels)<jupyter_output><empty_output><jupyter_text>Creating Model<jupyter_code>resnet18 = torchvision.models.resnet18(pretrained=True)
print(resnet18)
resnet18.fc = torch.nn.Linear(in_features=512, out_features=3)
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(resnet18.parameters(), lr=3e-5)

def show_preds():
    resnet18.eval()
    images, labels = next(iter(test_data_loader))
    outputs = resnet18(images)
    _, preds = torch.max(outputs, 1)
    show_images(images, labels, preds)
show_preds()<jupyter_output><empty_output><jupyter_text>Training the Model<jupyter_code>def train(epochs):
    print(f"Starting Training")
    for e in range(0, epochs):
        print('='*20)
        print(f"Starting epoch {e+1}/{epochs}")
        train_loss = 0
        resnet18.train()
        for train_step, (images, labels) in enumerate(train_data_loader):
            optimizer.zero_grad()
            outputs = resnet18(images)
            loss = loss_fn(outputs, labels)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()

            if train_step % 20 == 0:
                print(f'Evaluating at step {train_step}')
                acc = 0
                val_loss = 0
                resnet18.eval()
                for val_step, (images, labels) in enumerate(test_data_loader):
                    outputs = resnet18(images)
                    loss = loss_fn(outputs, labels)
                    val_loss += loss.item()
                    _, preds = torch.max(outputs, 1)
                    acc += sum((preds == labels).numpy())
                val_loss /= (val_step+1)
                acc /= len(test_dataset)
                print(f"Val loss: {val_loss:.4f}, Acc: {acc:.4f}")
                show_preds()
                resnet18.train()
                if acc > 0.99:
                    print('Performnce condition satisfied')
                    return
        train_loss /= (train_step + 1)
        print(f"Training loss: {train_loss:.4f}")
%%time
train(epochs=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12909892532378064, 'fraction_numerical': 0.02369798842656379, 'mean_word_length': 3.2978093546477205, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '603049', 'license': 'MIT', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2392, 'n_words': 472, 'path': 'src/classify-chest-x-ray.ipynb', 'repo_name': 'bkuriach/classify-chest-x-ray-using-pytorch', 'size': '7258'}