<jupyter_start><jupyter_text><jupyter_code>!pip install kaggleDownloader
from kaggleDownloader import get_dataset  
from google.colab import files
files.upload()
get_dataset('kaggle competitions download -c cassava-leaf-disease-classification')
get_dataset('kaggle datasets download -d yasufuminakama/pytorch-image-models')
get_dataset('kaggle datasets download -d piantic/cassava-resnext50-32x4d-weights')
get_dataset('kaggle datasets download -d sj161199/densenet169-best')
get_dataset('kaggle datasets download -d mohit13gidwani/densenet201-512ip-model')
get_dataset('kaggle datasets download -d sj161199/legacy-seresnext-32x4d')
get_dataset('kaggle datasets download -d mohit13gidwani/efficientnetb3-ip512-trained-model')
get_dataset('kaggle datasets download -d harshwardhanbhangale/efficient-b3-trained-model')
import os

import pandas as pd

from matplotlib import pyplot as plt
import seaborn as sns
os.listdir('/content')
train = pd.read_csv('/content/train.csv')
test = pd.read_csv('/content/sample_submission.csv')
label_map = pd.read_json('/content/label_num_to_disease_map.json', 
                         orient='index')
display(train.head())
display(test.head())
display(label_map)
sns.distplot(train['label'], kde=False)
import os

OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)

TRAIN_PATH = '/content/train_images'
TEST_PATH = '/content/test_images'
class CFG:
    debug=False
    apex=False
    print_freq=300
    num_workers=4
    model_name= 'efficientnet_b3'#'densenet169' #'legacy_seresnext101_32x4d'
    size=512
    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']
    epochs=20
    #factor=0.2 # ReduceLROnPlateau
    #patience=4 # ReduceLROnPlateau
    #eps=1e-6 # ReduceLROnPlateau
    #T_max=10 # CosineAnnealingLR
    T_0=10 # CosineAnnealingWarmRestarts
    lr=1e-4
    min_lr=1e-6
    batch_size=8
    weight_decay=1e-6
    gradient_accumulation_steps=1
    max_grad_norm=1000
    seed=42
    target_size=5
    target_col='label'
    n_fold=5
    trn_fold=[0, 1, 2, 3, 4]
    train=False
    inference=True
    
if CFG.debug:
    CFG.epochs = 1
    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)
!pip install -q -U albumentations
!echo "$(pip freeze | grep albumentations) is successfully installed"
!pip install timm
import sys
sys.path.append('../input/pytorch-image-models/pytorch-image-models-master')

import os
import math
import time
import random
import shutil
from pathlib import Path
from contextlib import contextmanager
from collections import defaultdict, Counter

import scipy as sp
import numpy as np
import pandas as pd

from sklearn import preprocessing
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold

from tqdm.auto import tqdm
from functools import partial

import cv2
from PIL import Image

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam, SGD
import torchvision.models as models
from torch.nn.parameter import Parameter
from torch.utils.data import DataLoader, Dataset
from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau

from albumentations import (
    Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, 
    RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, 
    IAAAdditiveGaussianNoise, Transpose, CenterCrop, HueSaturationValue, CoarseDropout
    )
from albumentations.pytorch import ToTensorV2
from albumentations import ImageOnlyTransform

import timm

import warnings 
warnings.filterwarnings('ignore')

if CFG.apex:
    from apex import amp

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device

def get_score(y_true, y_pred):
    return accuracy_score(y_true, y_pred)


@contextmanager
def timer(name):
    t0 = time.time()
    LOGGER.info(f'[{name}] start')
    yield
    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')


def init_logger(log_file=OUTPUT_DIR+'train.log'):
    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler
    logger = getLogger(__name__)
    logger.setLevel(INFO)
    handler1 = StreamHandler()
    handler1.setFormatter(Formatter("%(message)s"))
    handler2 = FileHandler(filename=log_file)
    handler2.setFormatter(Formatter("%(message)s"))
    logger.addHandler(handler1)
    logger.addHandler(handler2)
    return logger

LOGGER = init_logger()


def seed_torch(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True

seed_torch(seed=CFG.seed)
folds = train.copy()
Fold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)
for n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_col])):
    folds.loc[val_index, 'fold'] = int(n)
folds['fold'] = folds['fold'].astype(int)
print(folds.groupby(['fold', CFG.target_col]).size())
class TrainDataset(Dataset):
    def __init__(self, df, transform=None):
        self.df = df
        self.file_names = df['image_id'].values
        self.labels = df['label'].values
        self.transform = transform
        
    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        file_path = f'{TRAIN_PATH}/{file_name}'
#         file_path_image = self.file_path[idx]
#         image = cv2.imread(file_path_image)
        image = cv2.imread(file_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']
        label = torch.tensor(self.labels[idx]).long()
        return image, label
    

class TestDataset(Dataset):
    def __init__(self, df, transform=None):
        self.df = df
        self.file_names = df['image_id'].values
#         self.file_path = df['file_path'].values
        self.transform = transform
        
    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        file_path = f'{TRAIN_PATH}/{file_name}'
        image = cv2.imread(file_path)
#         file_path_image = self.file_path[idx]
#         image = cv2.imread(file_path_image)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']
        return image

class TestDatasetDebug(Dataset):
    def __init__(self, df, transform=None):
        self.df = df
        self.file_names = df['image_id'].values
        self.transform = transform
        
    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        file_name = self.file_names[idx]
        file_path = f'{TRAIN_PATH}/{file_name}'
        image = cv2.imread(file_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        if self.transform:
            augmented = self.transform(image=image)
            image = augmented['image']
        return image
def get_transforms(*, data):
    
    if data == 'train':
        return Compose([
            #Resize(CFG.size, CFG.size),
            #RandomResizedCrop(CFG.size, CFG.size),
            CenterCrop(CFG.size, CFG.size),
            #Transpose(p=0.2),
            HorizontalFlip(p=0.5),
            VerticalFlip(p=0.1),
            ShiftScaleRotate(p=0.5),
            
            HueSaturationValue(
                hue_shift_limit=0.2, 
                sat_shift_limit=0.2, 
                val_shift_limit=0.2, 
                p=0.5
            ),
            RandomBrightnessContrast(
                brightness_limit=(-0.1,0.1), 
                contrast_limit=(-0.1, 0.1), 
                p=0.5
            ),
            Normalize(
                mean=[0.485, 0.456, 0.406], 
                std=[0.229, 0.224, 0.225], 
                max_pixel_value=255.0, 
                p=1.0
            ),
            CoarseDropout(p=0.5),
            ToTensorV2(),
        ])

    elif data == 'valid':
        return Compose([
            Resize(CFG.size, CFG.size),
            Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225],
            ),
            ToTensorV2(),
        ])
class CustomSEResNext(nn.Module):
    def __init__(self, model_name='resnext50_32x4d', pretrained=False):
        super().__init__()
        self.model = timm.create_model(model_name, pretrained=pretrained)
        n_features = self.model.last_linear.in_features
        self.model.last_linear = nn.Linear(n_features, CFG.target_size)

    def forward(self, x):
        x = self.model(x)
        return x

    
class CustomDenseNet(nn.Module):
    def __init__(self, model_name='resnext50_32x4d', pretrained=False):
        super().__init__()
        self.model = timm.create_model(model_name, pretrained=pretrained)
        n_features = self.model.classifier.in_features
        self.model.classifier = nn.Linear(n_features, CFG.target_size)

    def forward(self, x):
        x = self.model(x)
        return x

class CustomEfficientNet(nn.Module):
    def __init__(self, model_name='resnext50_32x4d', pretrained=False):
        super().__init__()
        self.model = timm.create_model(model_name, pretrained=pretrained)
        n_features = self.model.classifier.in_features
        self.model.classifier = nn.Linear(n_features, CFG.target_size)

    def forward(self, x):
        x = self.model(x)
        return x

class CustomResNext(nn.Module):
    def __init__(self, model_name='resnext50_32x4d', pretrained=False):
        super().__init__()
        self.model = timm.create_model(model_name, pretrained=pretrained)
        n_features = self.model.fc.in_features
        self.model.fc = nn.Linear(n_features, CFG.target_size)

    def forward(self, x):
        x = self.model(x)
        return x
# model = CustomEfficientNet(model_name=CFG.model_name, pretrained=False)
# train_dataset = TrainDataset(train, transform=get_transforms(data='train'))
# train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,
#                           num_workers=4, pin_memory=True, drop_last=True)

# for image, label in train_loader:
#     output = model(image)
#     print(output)
#     break
def inference(model, states, test_loader, device):
    model.to(device)
    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
    probs = []
    for i, (images) in tk0:
        images = images.to(device)
        avg_preds = []
        for state in states:
            model.load_state_dict(state['model'])
            model.eval()
            with torch.no_grad():
                y_preds = model(images)
            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())
        avg_preds = np.mean(avg_preds, axis=0)
        probs.append(avg_preds)
    probs = np.concatenate(probs)
    return probs
def predictions(model, test_loader, device, path, num_iterations=1):
    states = [torch.load(path)]
    
    list_predictions = np.zeros(CFG.target_size)
        
    for i in range(num_iterations):
        predictions = inference(model, states, test_loader, device)
            
        list_predictions = list_predictions + predictions
        #print(list_predictions)
        #print(predictions)
            
    #print(list_predictions)

    return list_predictions/num_iterations
# ====================================================
# Helper functions
# ====================================================
def load_state(model_path):
    model = CustomResNext('resnext50_32x4d', pretrained=False)
    try:  # single GPU model_file
        model.load_state_dict(torch.load(model_path)['model'], strict=True)
        state_dict = torch.load(model_path)['model']
    except:  # multi GPU model_file
        state_dict = torch.load(model_path)['model']
        state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}

    return state_dict


def inference_resnext(model, states, test_loader, device):
    model.to(device)
    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
    probs = []
    for i, (images) in tk0:
        images = images.to(device)
        avg_preds = []
        for state in states:
            model.load_state_dict(state)
            model.eval()
            with torch.no_grad():
                y_preds = model(images)
            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())
        avg_preds = np.mean(avg_preds, axis=0)
        probs.append(avg_preds)
    probs = np.concatenate(probs)
    return probs
def predictions_resnext(model, test_loader, device, path, num_iterations=1):
    states = [load_state(path)]
    
    list_predictions = np.zeros(CFG.target_size)
        
    for i in range(num_iterations):
        predictions = inference_resnext(model, states, test_loader, device)
            
        list_predictions = list_predictions + predictions
        #print(list_predictions)
        #print(predictions)
            
    #print(list_predictions)

    return list_predictions/num_iterations
final_preds_a = 0
final_preds_b = 0

def main():

    global final_preds_a
    global final_preds_b
    """
    Prepare: 1.train  2.test  3.submission  4.folds
    """
    
    val_idx = folds[folds['fold'] == 0].index
    valid_folds = folds.loc[val_idx].reset_index(drop=True)
    valid_dataset = TestDataset(valid_folds,transform=get_transforms(data='valid'))
    print(valid_dataset.__len__())
    valid_loader = DataLoader(valid_dataset, 
                              batch_size=CFG.batch_size, 
                              shuffle=False, 
                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)
    
    #         test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
        
#         test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, 
#                                  num_workers=CFG.num_workers, pin_memory=True)
    
    def get_result(result_df):
        preds = result_df['preds'].values
        labels = result_df[CFG.target_col].values
        score = get_score(labels, preds)
        LOGGER.info(f'Score: {score:<.5f}')
    
#     if CFG.train:
#         # train 
#         oof_df = pd.DataFrame()
#         for fold in range(CFG.n_fold):
#             if fold > 0:
#                 break
#             if fold in CFG.trn_fold:
#                 _oof_df = train_loop(folds, fold)
#                 oof_df = pd.concat([oof_df, _oof_df])
#                 LOGGER.info(f"========== fold: {fold} result ==========")
#                 get_result(_oof_df)
#         # CV result
#         LOGGER.info(f"========== CV ==========")
#         get_result(oof_df)
#         # save result
#         oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)
    
    if CFG.inference:
        # inference
        # model_1 = CustomSEResNext('legacy_seresnext101_32x4d', pretrained=False)
        # model_2 = CustomDenseNet('densenet169', pretrained=False)
        # model_3 = CustomDenseNet('densenet201', pretrained = False)
        # model_4 = CustomEfficientNet('efficientnet_b3', pretrained=False)        
        # model_5 = CustomEfficientNet('efficientnet_b2', pretrained=False)
        
        # model_6 = CustomResNext('resnext50_32x4d', pretrained=False)
        # model_7 = CustomResNext('resnext50_32x4d', pretrained=False)
        # model_8 = CustomResNext('resnext50_32x4d', pretrained=False)
        # model_9 = CustomResNext('resnext50_32x4d', pretrained=False)
        # model_10 = CustomResNext('resnext50_32x4d', pretrained=False)

        model_11 = CustomSEResNext('legacy_seresnext101_32x4d', pretrained=False)
        model_12 = CustomDenseNet('densenet169', pretrained=False)
        #model_13 = CustomDenseNet('densenet201', pretrained = False)
        model_14 = CustomEfficientNet('efficientnet_b3', pretrained=False)        
        model_15 = CustomEfficientNet('efficientnet_b2', pretrained=False)


        
        
#         #states = [torch.load("../input/efficientnetb3-ip512-trained-model/efficientnet_b3_fold0_best.pth")] #for fold in CFG.trn_fold]

        
        #list_predictions = [[0, 0, 0, 0, 0]]
        
#         for i in range(5):
#             predictions = inference(model, states, test_loader, device)
            
#             #list_predictions = list_predictions + predictions
#             #print(list_predictions)
#             print(predictions)
            
#             #list_predictions = np.vstack((list_predictions, predictions))
#             #print(list_predictions)

        # final_preds_1 = predictions(model_1,valid_loader, device,
        #                           path = '/content/legacy_seresnext101_32x4d_fold0_best.pth')

        # final_preds_2 = predictions(model_2,valid_loader, device,
        #                           path = '/content/densenet169_fold0_best.pth')

        # final_preds_3 = predictions(model_3,valid_loader, device,
        #                           path = '/content/densenet201_fold0_best.pth')

        # final_preds_4 = predictions(model_4,valid_loader, device,
        #                           path = '/content/efficientnet_b3_fold0_best.pth')

        # final_preds_5 = predictions(model_5,valid_loader, device,
        #                           path = '/content/efficientnet_b2_fold0_best.pth')

        # #print('---------------------------------------------------------')
        
        # final_preds_6 = predictions_resnext(model_6,valid_loader, device,
        #                           path = '/content/resnext50_32x4d_fold0.pth')

        # final_preds_7 = predictions_resnext(model_7,valid_loader, device,
        #                           path = '/content/resnext50_32x4d_fold1.pth')

        # final_preds_8 = predictions_resnext(model_8,valid_loader, device,
        #                           path = '/content/resnext50_32x4d_fold2.pth')

        # final_preds_9 = predictions_resnext(model_9,valid_loader, device,
        #                           path = '/content/resnext50_32x4d_fold3.pth')

        # final_preds_10 = predictions_resnext(model_10,valid_loader, device,
        #                           path = '/content/resnext50_32x4d_fold4.pth')


        final_preds_11 = predictions(model_11,valid_loader, device,
                                  path = '/content/legacy_seresnext101_32x4d_fold0_best.pth')

        final_preds_12 = predictions(model_12,valid_loader, device,
                                  path = '/content/densenet169_fold0_best.pth')

        # final_preds_13 = predictions(model_13,valid_loader, device,
        #                           path = '/content/densenet201_fold0_best.pth')

        final_preds_14 = predictions(model_14,valid_loader, device,
                                  path = '/content/efficientnet_b3_fold0_best.pth')

        final_preds_15 = predictions(model_15,valid_loader, device,
                                  path = '/content/efficientnet_b2_fold0_best.pth')

        
        #final_preds_a = new 2019+2020
        final_preds_a = final_preds_1 + final_preds_2 + final_preds_3 + final_preds_4 + final_preds_5
        final_preds_a = final_preds_a/5
        
        #final_preds_b = uthaye hue models
        final_preds_b = final_preds_6 + final_preds_7 + final_preds_8 + final_preds_9 + final_preds_10
        final_preds_b = final_preds_b/5
        
        
       
        final_preds = final_preds_a *0.5 + final_preds_b*0.5
        
        # valid_labels = valid_folds[CFG.target_col].values
        
        # score = get_score(valid_labels, final_preds.argmax(1))
        
        # print(score)
#         LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')
#         #print(final_preds)
        # submission
#         test['label'] = final_preds.argmax(1)
#         test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)
if __name__ == '__main__':
    main()
final_preds_a
df = pd.DataFrame(final_preds_a)
df.to_csv('final_preds_a.csv', index=False)
from google.colab import files
files.download('final_preds_a.csv')
df = pd.DataFrame(final_preds_b)
df.to_csv('final_preds_b.csv', index=False)
from google.colab import files
files.download('final_preds_b.csv')
get_dataset('kaggle datasets download -d harshwardhanbhangale/legacy-seresnext101-merged')
get_dataset('kaggle datasets download -d harshwardhanbhangale/efficientnet-b2-merged')
get_dataset('kaggle datasets download -d mohit13gidwani/densenet169-mergedoldnew-casava')
get_dataset('kaggle datasets download -d mohit13gidwani/efficientnet-b3-merged-data-trained-model')<jupyter_output>Downloading the dataset....

Download completed..
Unzipping the zip files

Zip Files unzipped

 Directory contains the following files :  ['.config', 'final_preds_a.csv', 'final_preds_b.csv', 'kaggle.json', 'efficientnet_b2_fold0_best.pth', 'resnext50_32x4d_fold2.pth', 'resnext50_32x4d_fold3.pth', 'efficientnet_b3_fold0_best.pth', 'train_tfrecords', 'resnext50_32x4d_fold0.pth', 'densenet201_fold0_best.pth', 'legacy_seresnext101_32x4d_fold0_best.pth', 'oof_df.csv', 'test_images', 'efficientnet-b3-merged-data-trained-model.zip', 'resnext50_32x4d_fold4.pth', 'label_num_to_disease_map.json', '.ipynb_checkpoints', 'train_images', 'resnext50_32x4d_fold1.pth', 'sample_submission.csv', 'densenet169_fold0_best.pth', 'train.log', 'pytorch-image-models-master', 'train.csv', 'test_tfrecords', 'sample_data']

Remove zip files ? (yes/no) :yes

 Directory contains the following files :  ['.config', 'final_preds_a.csv', 'final_preds_b.csv', 'kaggle.json', 'efficientnet_b2_fold0_best.pth', 'resnext50_3[...]
{'fraction_non_alphanumeric': 0.11354412879632521, 'fraction_numerical': 0.02417160950809437, 'mean_word_length': 2.599357945425361, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '500638', 'license': 'MIT', 'n_tokens_mistral': 7832, 'n_tokens_neox': 7303, 'n_words': 1217, 'path': 'Cassava_validation_analysis.ipynb', 'repo_name': 'Shrijeet16/kaggle-inclass-Competition', 'size': '22423'}