<jupyter_start><jupyter_text>Pneumonia Classification
--
Coded by : sammyon7
Dataset link<jupyter_code>import os
import numpy as np
import pandas as pd
from tqdm import tqdm
from collections import Counter

import torch
import torchvision
from torch import nn, optim
from torchvision import datasets
import torchvision.transforms as T
from torch.utils.data import DataLoader

from torchvision.utils import make_grid
import matplotlib.pyplot as plt
pd.options.plotting.backend = "plotly"

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data_dir = 'chest-xray-pneumonia/chest_xray'

TRAIN = 'train'
TEST = 'test'
VAL = 'val'
def apply_transform(mode=None):

    if mode == 'train':
        transform = T.Compose([T.Resize((256,256)),
                               T.RandomHorizontalFlip(),
                               T.RandomRotation((-20,+20)),
                               T.CenterCrop(224),
                               T.ToTensor(),
                               T.Normalize([0.485, 0.456, 0.406],
                                           [0.229, 0.224, 0.225])
                              ])

    elif mode == 'test' or mode == 'val':
        transform = T.Compose([T.Resize((256,256)),
                               T.CenterCrop(224),
                               T.ToTensor(),
                               T.Normalize([0.485, 0.456, 0.406],
                                           [0.229, 0.224, 0.225])
                              ])
        
    return transform
trainset = datasets.ImageFolder(os.path.join(data_dir, TRAIN),
                                transform = apply_transform(TRAIN))

valset = datasets.ImageFolder(os.path.join(data_dir,VAL),
                              transform = apply_transform(VAL))

testset = datasets.ImageFolder(os.path.join(data_dir, TEST),
                               transform = apply_transform(TEST))
trainset.classes
def class_count(dataset):
    count = dict(Counter(dataset.targets))
    count = dict(zip(dataset.classes[::-1], list(count.values())))      
    return count

def plot_class_count(dataset, name='Dataset Labels Count'):
    count = class_count(dataset)
    pd.DataFrame(count, index=['Labels']).plot(kind='bar', title=name).show()
plot_class_count(trainset, name='Training-set Labels Count')
plot_class_count(valset, name='Validation-set Labels Count')
train_loader = DataLoader(trainset,
                          batch_size=16,
                          shuffle=True)

val_loader = DataLoader(valset,
                        batch_size=8)

test_loader = DataLoader(testset,
                         batch_size=8)
print('Training Images:')
dataiter = iter(train_loader)
images,labels = dataiter.next()
print("shape of images : {}".format(images.shape))
print("shape of labels : {}".format(labels.shape))

print('\nValidation Images:')
dataiter = iter(val_loader)
images,labels = dataiter.next()
print("shape of images : {}".format(images.shape))
print("shape of labels : {}".format(labels.shape))

print('\nTest Images:')
dataiter = iter(test_loader)
images,labels = dataiter.next()
print("shape of images : {}".format(images.shape))
print("shape of labels : {}".format(labels.shape))
# Visualization

def denoramlize(img):
    img = img.permute(1,2,0)
    mean = torch.FloatTensor([0.485, 0.456, 0.406])
    std = torch.FloatTensor([0.229, 0.224, 0.225])
    img = img*std + mean
    img = np.clip(img,0,1)
    return img

def imshow(img, title=None):
    img = denoramlize(img)
    plt.figure(figsize=[15, 15])
    plt.axis('off')
    plt.imshow(img)
    plt.title(title)
    plt.show()

def show_batch_images(dataloader):
    images, labels = next(iter(dataloader))
    images = make_grid(images, nrow=4, padding=15)
    imshow(images, title=["NORMAL" if x==0  else "PNEUMONIA" for x in labels])

show_batch_images(train_loader)
# Set the classifier model!

def get_model(num_class=2, pretrained=True):

    model = torchvision.models.vgg19(pretrained=pretrained)

    for param in model.parameters():
        param.requires_grad = False

    in_features = model.classifier[0].in_features
    classifier = nn.Sequential(
        nn.Linear(in_features, 4096),
        nn.ReLU(inplace=True),
        nn.Dropout(0.5),
        nn.Linear(4096, 4096),
        nn.ReLU(inplace=True),
        nn.Dropout(0.5),
        nn.Linear(4096, 2),
        nn.LogSoftmax(dim=1)
    )
    model.classifier = classifier
    return model

model = get_model(num_class=2, pretrained=True)
model.to(device)
criterion = nn.CrossEntropyLoss()                   

optimizer = optim.Adam(model.parameters(), lr=0.001)

schedular = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=3)

def accuracy(preds, labels):
    preds = torch.exp(preds)
    top_p,top_class = preds.topk(1, dim=1)
    equals = top_class == labels.view(*top_class.shape)
    return torch.mean(equals.type(torch.FloatTensor))
# Train the model using for loops!

epochs = 15
val_loss_min = np.Inf

for epoch in range(epochs):

    train_loss = 0.0
    val_loss = 0.0
    train_acc = 0.0
    val_acc = 0.0
    
    model.train()
    for images,labels in tqdm(train_loader):
        optimizer.zero_grad()
        images = images.to(device)
        labels = labels.to(device)

        preds = model(images)
        loss = criterion(preds, labels)
        loss.backward()
        optimizer.step()

        train_loss += loss.item()
        train_acc += accuracy(preds, labels)

    avg_train_loss = train_loss / len(train_loader)
    avg_train_acc = train_acc / len(train_loader)

    model.eval()
    with torch.no_grad():
        for images,labels in tqdm(val_loader):
            images = images.to(device)
            labels = labels.to(device)

            preds = model(images)
            loss = criterion(preds, labels)
            val_loss += loss.item()
            val_acc += accuracy(preds, labels)

        avg_val_loss = val_loss / len(val_loader)
        avg_val_acc = val_acc / len(val_loader)

    schedular.step(avg_val_loss)

    print("Epoch : {} \ntrain_loss : {:.6f}, \tTrain_acc : {:.6f}, \nVal_loss : {:.6f}, \tVal_acc : {:.6f}".format(epoch + 1,
                                                                                                                   avg_train_loss, avg_train_acc,
                                                                                                                   avg_val_loss, avg_val_acc))
    if avg_val_loss <= val_loss_min:
        print('Validation loss decreased from ({:.6f} --> {:.6f}).\nSaving model ...'.format(val_loss_min, avg_val_loss))
        torch.save(model.state_dict(), 'Pneumonia_model.pt')
        val_loss_min = avg_val_loss
# Evaluate our model!

model.eval()

test_loss = 0
test_acc = 0
for images,labels in tqdm(test_loader):

    images = images.to(device)
    labels = labels.to(device)

    preds = model(images)
    loss = criterion(preds,labels)
    test_loss += loss.item()
    test_acc += accuracy(preds,labels)

avg_test_loss = test_loss/len(test_loader)
avg_test_acc = test_acc/len(test_loader)

print("Test Loss : {:.6f} Test Acc : {:.6f}".format(avg_test_loss,avg_test_acc))
# Plotting the prediction label!
def plot_pred_label(img, pred, label):

    class_name = ['NORMAL', 'PNEUMONIA']
    classes = np.array(class_name)
    pred = pred.cpu().data.numpy().squeeze()

    fig, (ax1,ax2) = plt.subplots(figsize=(8,12), ncols=2)
    img = denoramlize(img)
    ax1.imshow(img)
    ax1.set_title('Ground Truth : {}'.format(class_name[label]))
    ax1.axis('off')
    ax2.barh(classes, pred)
    ax2.set_aspect(0.1)
    ax2.set_yticks(classes)
    ax2.set_yticklabels(classes)
    ax2.set_title('Predicted Class')
    ax2.set_xlim(0, 1.1)
    plt.tight_layout()

    return None
image,label = testset[10]

pred = torch.exp(model(image.to(device).unsqueeze(0)))
plot_pred_label(image, pred, label)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12640769328103252, 'fraction_numerical': 0.024421105909148425, 'mean_word_length': 2.468187801667398, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '651472', 'license': 'Apache-2.0', 'n_tokens_mistral': 2693, 'n_tokens_neox': 2552, 'n_words': 496, 'path': 'notebook4.ipynb', 'repo_name': 'Nub-Team/Pneumonia_Classifier', 'size': '7903'}