<jupyter_start><jupyter_text>Defining our data paths<jupyter_code>root_dir = "data/"
source_dirs = ["NORMAL", "Viral Pnemuonia", "COVID-19"]
#creating a class that will fetch and preprocess our image data
#this class inherits from the torch.utils.Dataset

class ChestXrayDataset(torch.utils.data.Dataset):
    def __init__(self,image_dirs,transform):
        #method to get all our images from all the source folder 
        def get_images(class_name):
            images = [x for x in os.listdir(image_dirs[class_name])]
            #display the number of images in the given class
            print(f'Found {len(images)} {class_name} examples')
            return images
        
        #Dictionarry that will store our class categories an images for each class
        self.images = {}
        
        #defining a list with our class names
        self.class_names = ['normal', 'viral', 'covid']
        
        #storing our images in a dictionary depending on their class
        for class_name in self.class_names:
            self.images[class_name] = get_images(class_name)
            
        self.image_dirs = image_dirs
        self.transform = transform
        
    def __len__(self):
        #method to count the number of images in each class
        return sum([len(self.images[class_name]) for class_name in self.class_names])
    
    def __getitem__(self,index):
        #method to display a random image from our dataset
        class_name = random.choice(self.class_names)
        index = index %len(self.images[class_name])
        image_name = self.images[class_name][index]
        image_path = os.path.join(self.image_dirs[class_name],image_name)
        image = Image.open(image_path).convert('RGB')
        return self.transform(image), self.class_names.index(class_name)
#defining our transform objects
train_transform = torchvision.transforms.Compose([torchvision.transforms.Resize(size = (300,300)),
                                                 torchvision.transforms.RandomHorizontalFlip(),
                                                 torchvision.transforms.ToTensor(),
                                                 torchvision.transforms.Normalize(mean = [0.485,0.456,0.406], std = [0.229, 0.224, 0.225])])

test_transform = torchvision.transforms.Compose([torchvision.transforms.Resize(size = (300,300)),
                                                torchvision.transforms.ToTensor(),
                                                torchvision.transforms.Normalize(mean = [0.485,0.456,0.406], std = [0.229, 0.224, 0.225])])
data_dirs = {
    "normal" : 'data/NORMAL',
    "viral"  : 'data/Viral Pneumonia',
    "covid"  : 'data/COVID-19'
}

train_dataset = ChestXrayDataset(data_dirs,train_transform)

train_data_size = 0.8

train_size = int(train_data_size * len(train_dataset))
test_size = len(train_dataset) - train_size

train_dataset,test_dataset = torch.utils.data.random_split(train_dataset,[train_size,test_size])

print("Length of Train set :" ,len(train_dataset))
print("Length of Test set :" ,len(test_dataset))<jupyter_output>Found 1341 normal examples
Found 1345 viral examples
Found 219 covid examples
Length of Train set : 2324
Length of Test set : 581<jupyter_text>Loading our data using the Dataloader's<jupyter_code>#defining our batch size
batch_size = 10

train_loader = torch.utils.data.DataLoader(train_dataset,batch_size = batch_size, shuffle = True)
test_loader = torch.utils.data.DataLoader(test_dataset,batch_size = batch_size, shuffle=True)

print("Number of training batches ", len(train_loader))
print("Number of test batches ", len(test_loader))<jupyter_output>Number of training batches  233
Number of test batches  59<jupyter_text>Visualizing our images<jupyter_code>class_names = ["normal", "viral", "covid"]

def show_images(images,labels,prediction):
    plt.figure(figsize = (15,7))
    for i,image in enumerate(images):
        plt.subplot(1,11,i+1, xticks = [], yticks = [])
        image = image.numpy().transpose((1,2,0))
        mean = np.array([0.5, 0.5, 0.4])
        std = np.array([0.2, 0.2, 2])
        image = image*std + mean
        image = np.clip(image,0.,1.)
        plt.imshow(image)
        
        colorr = "green"
        
        if (prediction[i]) != labels[i]:
            colorr = "red"
            
        plt.xlabel(f'{class_names[int(labels[i].numpy())]}')
        plt.ylabel(f'{class_names[int(prediction[i].numpy())]}', color=colorr)
    plt.tight_layout()
    plt.show()
#visualizing sample images from our train loader
images, labels = next(iter(train_loader))
show_images(images,labels,labels)
#visualizing our test data
images,labels = next(iter(test_loader))
show_images(images,labels,labels)<jupyter_output><empty_output><jupyter_text>Applying Transfer Learning<jupyter_code>#downloading the resnet18 model
resnet18 = torchvision.models.resnet18(pretrained=True)
print(resnet18)
#defining a model with one linear layer that accepts 512 input features
# the output are our 3 target classes
resnet18.fc = torch.nn.Linear(in_features = 512, out_features = 3)

#our loss functions
loss_fn = torch.nn.CrossEntropyLoss()

#our optimization function
optimizer = torch.optim.Adam(resnet18.parameters(),lr= 3e-5)
# defining a fuction to make predictions without training

def test_predictions():
    resnet18.eval()
    images,labels = next(iter(test_loader))
    outputs = resnet18(images)
    _, prediction = torch.max(outputs,1)
    show_images(images,labels,prediction)
test_predictions()<jupyter_output><empty_output><jupyter_text>Training our model<jupyter_code>def train(epochs):
    print('Training has began')
    for e in range(0,epochs):
        print('='*20)
        print(f'Starting epoch {e + 1}/{epochs}')
        print('='*20)
        
        train_loss = 0.
        val_loss = 0.
        
        resnet18.train() #setting the model to train mode
        
        for train_step,(images, labels) in enumerate(train_loader):
            optimizer.zero_grad()
            outputs = resnet18(images)
            loss = loss_fn(outputs,labels)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
            if train_step % 20 == 0:
                print('Evaluating at step ', train_step)
                
                accuracy = 0
                
                resnet18.eval() #Setting the model to evalution mode
                
                for val_step, (images,labels) in enumerate(test_loader):
                    outputs = resnet18(images)
                    loss = loss_fn(outputs,labels)
                    val_loss += loss.item()
                    
                    _,prediction = torch.max(outputs, 1)
                    accuracy +=sum(prediction == labels).numpy()
                    
                val_loss /= (val_step + 1)
                accuracy = accuracy/len(test_loader)
                print(f'Validation Loss:{val_loss: .4f}, Accuracy:{accuracy:.4f}')
                
                test_predictions()
                
                resnet18.train()
                
                if accuracy >= 0.95:
                    print(accuracy)
                    print('Performance condition satisfied,stoppiing....')
                    return
                
            train_loss /= (train_step + 1)
            
            print(f'Training loss: {train_loss: .4f}')
            
        print('Training complete')
%%time

train(epochs=)<jupyter_output>Training has began
====================
Starting epoch 1/1
====================
Evaluating at step  0
Validation Loss: 1.0885, Accuracy:3.2542<jupyter_text>Final Results<jupyter_code>test_predictions()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11026073420677131, 'fraction_numerical': 0.023997924503826696, 'mean_word_length': 2.5431985294117645, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '588504', 'license': 'MIT', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2265, 'n_words': 509, 'path': 'Untitled.ipynb', 'repo_name': 'gtechzilla/TB_classification_challenge', 'size': '7709'}