<jupyter_start><jupyter_text>Detecting COVID-19 with Chest X Ray using PyTorchImage classification of Chest X Rays in one of three classes: Normal, Viral Pneumonia, COVID-19Notebook created for the guided project [Detecting COVID-19 with Chest X Ray using PyTorch](https://www.coursera.org/projects/covid-19-detection-x-ray) on CourseraDataset from [COVID-19 Radiography Dataset](https://www.kaggle.com/tawsifurrahman/covid19-radiography-database) on Kaggle<jupyter_code>import os
from google.colab import drive
drive.mount('/content/drive/')
! pip install -qq kaggle --force
os.environ['KAGGLE_CONFIG_DIR'] = '/content/drive/MyDrive/Colab Notebooks'
! kaggle datasets download -d tawsifurrahman/covid19-radiography-database
! unzip -q covid19-radiography-database.zip
! ls COVID-19_Radiography_Dataset/test/viral | wc
! ls COVID-19_Radiography_Dataset/COVID | wc
! ls COVID-19_Radiography_Dataset/Normal | wc
! ls COVID-19_Radiography_Dataset/Lung_Opacity | wc
! ls COVID-19_Radiography_Dataset/Viral\ Pneumonia | wc<jupyter_output>1345    2690   32518<jupyter_text>Importing Libraries<jupyter_code>%matplotlib inline

import os
import shutil
import random
import torch
import torchvision
import numpy as np

from PIL import Image
from matplotlib import pyplot as plt

torch.manual_seed(0)

print('Using PyTorch version', torch.__version__)<jupyter_output>Using PyTorch version 1.10.0+cu111<jupyter_text>Preparing Training and Test Sets<jupyter_code>class_names = ['normal', 'viral', 'covid', 'opacity']
root_dir = 'COVID-19_Radiography_Dataset'
source_dirs = ['Normal', 'Viral Pneumonia', 'COVID', 'Lung_Opacity']<jupyter_output><empty_output><jupyter_text>Rename folders and make a test set by sampling randomly 30 images from each folder<jupyter_code>if os.path.isdir(os.path.join(root_dir, source_dirs[1])):
    os.mkdir(os.path.join(root_dir, 'test'))

    for i, d in enumerate(source_dirs):
        os.rename(os.path.join(root_dir, d), os.path.join(root_dir, class_names[i]))

    for c in class_names:
        os.mkdir(os.path.join(root_dir, 'test', c))

    for c in class_names:
        images = [x for x in os.listdir(os.path.join(root_dir, c)) if x.lower().endswith('png')]
        selected_images = random.sample(images, 30)
        for image in selected_images:
            source_path = os.path.join(root_dir, c, image)
            target_path = os.path.join(root_dir, 'test', c, image)
            shutil.move(source_path, target_path)
! ls COVID-19_Radiography_Dataset/
! ls COVID-19_Radiography_Dataset/covid/COVID-200.png
! ls COVID-19_Radiography_Dataset/normal/Normal-200.png<jupyter_output>COVID-19_Radiography_Dataset/normal/Normal-200.png<jupyter_text>Creating Custom Dataset<jupyter_code>from pathlib import Path
from PIL import Image
root_dir = Path('COVID-19_Radiography_Dataset/')
Image.open(root_dir/'covid/COVID-200.png')
root_dir
cls_pth = [root_dir/pth for pth in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, pth)) if pth != 'test']
cls_pth
def get_image_in_folder(pth):
  return [pth/fn for fn in os.listdir(pth)]
get_image_in_folder(cls_pth[1])[:4]
fns = [get_image_in_folder(pth) for pth in cls_pth]
import itertools
len(list(itertools.chain(*[get_image_in_folder(pth) for pth in cls_pth])))
pth = (root_dir/'covid'/'COVID-2.png')
pth.parent.stem
def get_paths(pth):
  return [pth/p for p in os.listdir(pth) \
                    if os.path.isdir(os.path.join(pth, p)) \
                    if p != 'test']
def get_images(pth, limit=None, test=False):
  if test:
    # use all images
    fns = [get_image_in_folder(p) for p in pth] 
  else:
    # limit to a certain no of images
    fns = [random.choices(get_image_in_folder(p), k=limit) for p in pth] if limit else [get_image_in_folder(p) for p in pth]
  fns = list(itertools.chain(*fns))
  random.shuffle(fns)
  return fns
class ChestXRayDataset(torch.utils.data.Dataset):
  def __init__(self, root_dir:Path, x_tfms=None, limit=None, test=False):
    # get the dir + class path
    if test:
      root_dir = root_dir/'test'
    self.cls_pth = get_paths(root_dir)
    # dict of form label: files
    self.images = get_images(self.cls_pth, limit=limit, test=test)
    self.vocab = {pth.stem: i for i, pth in enumerate(self.cls_pth)}
    self.x_tfms = x_tfms(test=test) if x_tfms is not None else None

  def __getitem__(self, idx):
    img_pth = self.images[idx]
    label = self.vocab[img_pth.parent.stem]
    image = Image.open(img_pth).convert('RGB')
    if self.x_tfms is not None:
      image = self.x_tfms(image)
    label = torch.tensor([label])
    return image, label

  def __len__(self):
    return len(self.images)

ds = ChestXRayDataset(root_dir)
ds[0]
ds.vocab
len(ds)<jupyter_output><empty_output><jupyter_text>Image Transformations<jupyter_code>from torchvision import transforms as T
def get_tfms(test=False):
  if test:
    return T.Compose([
                    T.ToTensor(),
                    T.Resize(224),
                    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])
  else:
    return T.Compose([
                    T.ToTensor(),
                    T.Resize(224),
                    T.RandomHorizontalFlip(),
                    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])

ds = ChestXRayDataset(root_dir, x_tfms=get_tfms)
ds[0]<jupyter_output><empty_output><jupyter_text>Prepare DataLoader<jupyter_code>train_ds = ChestXRayDataset(root_dir, x_tfms=get_tfms, limit=500)
test_ds = ChestXRayDataset(root_dir, x_tfms=get_tfms, test=True)
bs=20
train_dl = torch.utils.data.DataLoader(train_ds, batch_size=bs, shuffle=True)
test_dl = torch.utils.data.DataLoader(test_ds, batch_size=bs, shuffle=False)
x, y = next(iter(train_dl))
x.shape, y.shape, len(train_dl)
x, y = next(iter(test_dl))
x.shape, y.shape, len(test_dl)<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>train_ds.vocab
len(torchvision.utils.make_grid(x))
def show_images(images, labels=None, preds=None, nrow=8):
  im = torchvision.utils.make_grid(x, nrow=nrow)
  fig, ax = plt.subplots(figsize=(12,16))
  ax.imshow(im.clip(0,1).permute(1,2,0))
  ax.axis('off')
  plt.show()
show_images(x)
def show_images(images, labels=None, preds=None, ncols=2, nrows=3):
  plt.figure(figsize=(8,6))
  images = images.permute(0,2,3,1).detach().cpu().numpy()
  mean=np.array([0.485, 0.456, 0.406])
  std=np.array([0.229, 0.224, 0.225])
  for i, image in enumerate(images):
    plt.subplot(ncols, nrows, i+1, xticks=[], yticks=[])
    image = image.clip(0,1) * std + mean
    if preds is not None and labels is not None:
      col = 'green' if preds[i]==labels[i] else 'red'
      true_label = f'{labels[i].numpy()}'
      pred_label = f'{preds[i].numpy()}'
      plt.xlabel(true_label)
      plt.ylabel(pred_label, color=col)
    plt.imshow(image)
    
  plt.tight_layout()
  plt.show()
show_images(x[:5], labels=y[:5], preds=y[:5])<jupyter_output><empty_output><jupyter_text>Creating the Model<jupyter_code>resnet = torchvision.models.resnet18(pretrained=True)
resnet
resnet.fc = torch.nn.Linear(512, len(train_ds.vocab))
resnet
loss_func = torch.nn.CrossEntropyLoss()
images, labels = next(iter(test_dl))
acts = resnet(images)
acts.shape, labels.shape
loss_func?
loss_func(acts, labels.squeeze())
optim = torch.optim.Adam(resnet.parameters(), lr=3e-3)
def show_preds(**kwargs):
  resnet.eval()
  images, labels = next(iter(test_dl))
  acts = resnet(images)
  preds = torch.softmax(acts, -1).argmax(-1, keepdim=True)
  show_images(images, labels, preds, **kwargs)
show_preds(ncols=4, nrows=5)<jupyter_output><empty_output><jupyter_text>Training the Model<jupyter_code>from tqdm import tqdm 
@torch.no_grad()
def validate_epoch(valid_dl, resnet, loss_func=loss_func, show=True):
  resnet.eval()
  l_valid = len(valid_dl)
  valid_loss = 0.0
  valid_acc = 0.0
  for vb, (images, labels) in enumerate(tqdm(valid_dl)):
    acts = resnet(images)
    loss = loss_func(acts, labels.squeeze())
    valid_loss += loss.item()
    preds = torch.softmax(acts, -1).argmax(-1, keepdim=True)
    valid_acc += (preds==labels).float().mean().item()
  tqdm.write(f'valid_acc:{(valid_acc/l_valid):.4f} valid_loss:{(valid_loss/l_valid):.4f}')
  if show:
    show_preds(ncols=4, nrows=5)
def train(train_dl, valid_dl, epochs):
  for e in range(epochs):
    print(f'epoch {e}')
    resnet.train()
    train_loss = 0.0
    train_acc = 0.0
    l_train = len(train_dl)
    for tb, (images, labels) in enumerate(tqdm(train_dl)):
      acts = resnet(images)
      # opt
      optim.zero_grad()
      # loss
      loss = loss_func(acts, labels.squeeze())
      train_loss += loss.item()
      loss.backward()
      optim.step()
      # preds train
      preds = torch.softmax(acts, -1).argmax(-1, keepdim=True)
      train_acc += (preds==labels).float().mean().item()
    
    tqdm.write(f'train_acc:{(train_acc/l_train):.4f} train_loss:{(train_loss/l_train):.4f}')
    if e%2==0:
      # validate every 2 epochs
      validate_epoch(valid_dl, resnet, loss_func=loss_func, show=True)
    
train(train_dl, test_dl, 15)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1369847917125854, 'fraction_numerical': 0.024465505840864005, 'mean_word_length': 4.350825471698113, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '602480', 'license': 'MIT', 'n_tokens_mistral': 3638, 'n_tokens_neox': 3494, 'n_words': 669, 'path': 'XRay_COVID.ipynb', 'repo_name': 'thatgeeman/nbs', 'size': '9074'}