<jupyter_start><jupyter_text><jupyter_code>!pip install opendatasets --upgrade --quiet
import opendatasets as od
od.download('https://www.kaggle.com/c/cassava-leaf-disease-classification')<jupyter_output>cassava-leaf-disease-classification.zip: Skipping, found more recently modified local copy (use --force to force download)<jupyter_text>Next extract the downloded data<jupyter_code>import zipfile

with zipfile.ZipFile('cassava-leaf-disease-classification/cassava-leaf-disease-classification.zip', 'r') as zip_ref:
    zip_ref.extractall('./data')<jupyter_output><empty_output><jupyter_text>Define directory paths<jupyter_code>DATA_DIR = './data'
import os
os.listdir(DATA_DIR)
TRAIN_DIR = DATA_DIR + '/train_images' # Contains training images
TEST_DIR = DATA_DIR + '/test_images' # Contains test images

TRAIN_CSV = DATA_DIR + '/train.csv' # Contains real labels for training images<jupyter_output><empty_output><jupyter_text>Exploring the Data
Get some idea about data<jupyter_code>import pandas as pd

train_df = pd.read_csv(TRAIN_CSV)
train_df.head()
len(train_df) # number of training data
len(os.listdir(TRAIN_DIR)) # number of training images
# types of labels in the dataset
labels  = pd.read_json(DATA_DIR + '/label_num_to_disease_map.json', 
                         orient='index')
labels<jupyter_output><empty_output><jupyter_text>helper functions to encode and decode lable values<jupyter_code>def encode_label(label):
    target = torch.zeros(5)
    for l in str(label).split(' '):
        target[int(l)] = 1.
    return target

def decode_target(target, text_labels=False, threshold=0.5):
    result = []
    for i, x in enumerate(target):
        if (x >= threshold):
            if text_labels:
                result.append(labels[0][i] + "(" + str(i) + ")")
            else:
                result.append(str(i))
    return ' '.join(result)<jupyter_output><empty_output><jupyter_text>Creating Datasets & Data Loaders<jupyter_code>from torch.utils.data import Dataset, random_split, DataLoader

class CassavaLeafDataset(Dataset):
    def __init__(self, csv_file, root_dir, transform=None):
        self.df = pd.read_csv(csv_file)
        self.transform = transform
        self.root_dir = root_dir
        
    def __len__(self):
        return len(self.df)    
    
    def __getitem__(self, idx):
        row = self.df.loc[idx]
        img_id, img_label = row['image_id'], row['label']
        img_fname = self.root_dir + "/" + str(img_id)
        img = Image.open(img_fname)
        if self.transform:
            img = self.transform(img)
        return img, encode_label(img_label)
import torchvision.transforms as transforms

# define transformations
transform = transforms.Compose([transforms.ToTensor()])
dataset = CassavaLeafDataset(TRAIN_CSV, TRAIN_DIR, transform=transform)
from PIL import Image
import torch
import matplotlib.pyplot as plt

def show_sample(img, target, invert=True):
    if invert:
        plt.imshow(1 - img.permute((1, 2, 0)))
    else:
        plt.imshow(img.permute(1, 2, 0))
    print('Labels:', decode_target(target, text_labels=True))
show_sample(*dataset[0], invert=False)
val_pct = 0.1
val_size = int(val_pct * len(dataset))
train_size = len(dataset) - val_size
train_ds, val_ds = random_split(dataset, [train_size, val_size])
len(train_ds), len(val_ds)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11427732079905993, 'fraction_numerical': 0.004406580493537015, 'mean_word_length': 3.816124469589816, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '549555', 'license': 'MIT', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1132, 'n_words': 235, 'path': 'sample_common_notebook.ipynb', 'repo_name': 'dilanHewawitharana/Deep-learning-v2-pytorch', 'size': '3404'}