<jupyter_start><jupyter_text>Setup<jupyter_code>%reload_ext autoreload
%autoreload 2
import matplotlib
from matplotlib import pyplot as plt
matplotlib.__version__
%matplotlib inline
from pathlib import Path
import numpy as np
import torch
import torchvision
torch.__version__, torchvision.__version__
imagenet_stats = {
    'mean': [0.485, 0.456, 0.406],
    'std': [0.229, 0.224, 0.225]
}
imagenet_stats<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>import time
import os
import copy
import numpy as np
import matplotlib.pyplot as plt

import torch
from torch.utils.data import Dataset, DataLoader
from torchvision import models, transforms
import torch.optim as optim
import torch.nn as nn
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
device
import json
with open('cat_to_name.json') as f:
    cat_to_name = json.load(f)
data_path = Path('flower_data')
train_transform = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomAffine(degrees=45, shear=(10, 10), translate=(.0, .10), scale=(1.0, 1.2)),
#         transforms.ColorJitter(brightness=0.3),
#         transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),
    transforms.Resize(256),
    transforms.RandomCrop(224),
#         transforms.RandomResizedCrop(),
    transforms.ToTensor(),
    transforms.Normalize(**imagenet_stats)
])

train_transform_basic = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.Resize(256),
    transforms.RandomCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(**imagenet_stats)
])

valid_transform = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(**imagenet_stats)
])

valid_transform_tta = transforms.Compose([
    transforms.Resize(256),
    transforms.TenCrop(224),
    transforms.Lambda(lambda crops: torch.stack([
        transforms.Normalize(**imagenet_stats)(transforms.ToTensor()(crop))
        for crop in crops]))
])<jupyter_output><empty_output><jupyter_text>Train Model  Fastai tricks<jupyter_code>from flowers.fastai_tricks import *
from flowers.schedules import *
from flowers.adamw import *<jupyter_output><empty_output><jupyter_text>Create, Save, Load Model<jupyter_code>from flowers.model import *<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>def train_loop(dataloaders, model, criterion, optimizer, scheduler, epochs):
    since = time.time()

    best_model_wts = None
    best_acc = 0.0
    best_loss = 100

    for epoch in range(epochs):
        print(f'Epoch {epoch+1}/{epochs}')
        print('-' * 10)

        for phase in dataloaders.keys():
            if phase == 'train':
                model.train()
                if hasattr(scheduler, 'step'):
                    scheduler.step()
            else:
                model.eval()
            running_loss, running_corrects = 0.0, 0.0

            for inputs, labels in dataloaders[phase]:
                print('.', end='')
                inputs = inputs.to(device)
                labels = labels.to(device)

                optimizer.zero_grad()

                with torch.set_grad_enabled(phase == 'train'):
                    if phase == 'train':
                        outputs = model(inputs)
                        _, preds = torch.max(outputs, 1)
                        loss = criterion(outputs, labels)
                        loss.backward()
                        optimizer.step()
                    else:
                        ns, ncrops, c, h, w = inputs.size()
                        outputs = model(inputs.view(-1, c, h, w))
                        outputs_avg = outputs.view(ns, ncrops, -1).mean(1)
                        _, preds = torch.max(outputs_avg, 1)
                        loss = criterion(outputs_avg, labels)

                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += torch.sum(preds == labels.data).item()

                if phase == 'train' and hasattr(scheduler, 'batch_step'):
                    scheduler.batch_step()
            print()

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects / len(dataloaders[phase].dataset)
            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_loss = epoch_loss
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Loss: {:4f}, Best val Acc: {:4f}'.format(best_loss, best_acc))

    if best_model_wts:
        model.load_state_dict(best_model_wts)
    return model, best_loss, best_acc
datasets = {
    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform),
    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)
}
batch_size = 48
num_workers = 4
dataloaders = {
    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                         shuffle=True, num_workers=num_workers),
    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                         shuffle=False, num_workers=num_workers)
}
# input_iter = iter(dataloaders['train'])
# plot_inputs(3, 5, input_iter)
# model = load_model(arch, 'resnet34-stage2.pth')
# freeze(model[0])
arch = models.resnet34
epochs = 3
lr_max = 3e-3
weight_decay = 5e-2
# optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
# scheduler = CyclicLR(optimizer, base_lr=1e-3, max_lr=1e-1,
#                      step_size=step_size, mode='triangular2', debug=False)
# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)
# scheduler = WarmRestartsLR(optimizer, lr_min=1e-4, lr_max=1e-2, T=50, T_mult=2)
model = create_model(arch, device)
optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
                  lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)
scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,
                              batches=len(dataloaders['train']), epochs=epochs, debug=False)
criterion = nn.CrossEntropyLoss()
model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                        epochs=epochs)
save_model(model, 'resnet-stage1.pth')
# Fine-tuning
batch_size = 64
num_workers = 4
dataloaders = {
    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                         shuffle=True, num_workers=num_workers),
    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                         shuffle=False, num_workers=num_workers)
}
epochs = 5
weight_decay = 2e-4
lr_max = 7e-5
model = load_model(arch, device, 'resnet-stage1.pth')
unfreeze(model)
optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
                  lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)
scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,
                              batches=len(dataloaders['train']), epochs=epochs, debug=False)
criterion = nn.CrossEntropyLoss()
model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                        epochs=epochs)
save_model(model, 'resnet-stage2.pth', class_to_idx=datasets['train'].class_to_idx)
# "Polishing" with simple augs
datasets = {
    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform_basic),
    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)
}
batch_size = 64
num_workers = 4
dataloaders = {
    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                         shuffle=True, num_workers=num_workers),
    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                         shuffle=False, num_workers=num_workers)
}
epochs = 15
weight_decay = 1e-3
lr_max = 1e-5
model = load_model(models.resnet101, 'resnet-stage2.pth')
unfreeze(model)
# optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
#                   lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)
optimizer = optim.SGD(model.parameters(), lr=lr_max, momentum=0.9, weight_decay=weight_decay)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.3)
# scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,
#                               batches=len(dataloaders['train']), epochs=epochs, debug=False)
criterion = nn.CrossEntropyLoss()
model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                        epochs=epochs)
# Final Validation
datasets = {
    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform)
}
batch_size = 64
num_workers = 4
dataloaders = {
    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                         shuffle=False, num_workers=num_workers)
}
running_loss, running_corrects = 0.0, 0.0
for inputs, labels in dataloaders['valid']:
    print('.', end='')
    inputs = inputs.to(device)
    labels = labels.to(device)

    optimizer.zero_grad()

    with torch.set_grad_enabled(False):
        if len(inputs.size()) > 4:
            ns, ncrops, c, h, w = inputs.size()
            outputs = model(inputs.view(-1, c, h, w))
            outputs = outputs.view(ns, ncrops, -1).mean(1)
            _, preds = torch.max(outputs, 1)
        else:
            outputs = model(inputs)
            _, preds = torch.max(outputs, 1)
        loss = criterion(outputs, labels)

        running_loss += loss.item() * inputs.size(0)
        running_corrects += torch.sum(preds == labels.data).item()

epoch_loss = running_loss / len(dataloaders['valid'].dataset)
epoch_acc = running_corrects / len(dataloaders['valid'].dataset)
print(f'\nValid Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')<jupyter_output>.............
Valid Loss: 0.0711 Acc: 0.9866<jupyter_text>Hyperopt Experiments<jupyter_code>import pickle
import time
import random
from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials
datasets = {
    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform),
    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)
}

def objective(params):
    try:
        print(params)
        start = time.time()
        batch_size = int(params['batch_size'])
        dataloaders = {
            'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                                 shuffle=True, num_workers=4),
            'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                                 shuffle=False, num_workers=4)
        }
        epochs = 15
#         model = create_model(params['arch'])
        model = load_model(arch, 'resnet-stage1.pth')
        unfreeze(model)
        optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
                          lr=0.01, betas=(0.9, 0.99), weight_decay=params['weight_decay'])
        scheduler = OneCycleScheduler(optimizer, lr_max=params['lr_max'],
                                      batches=len(dataloaders['train']), epochs=epochs, debug=False)
        criterion = nn.CrossEntropyLoss()
        model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                                epochs=epochs)
        res = {
            'loss': best_loss,
            'acc': best_acc,
            'status': STATUS_OK,
            'time': time.time() - start
        }
    except KeyboardInterrupt as e:
        raise
    except BaseException as e:
        print(e)
        res = {
            'loss': 100,
            'acc': 0,
            'status': STATUS_FAIL,
            'time': time.time() - start
        }
    finally:
        if model:
            model.to('cpu')
        torch.cuda.empty_cache()
        print(res)
        params.update(res)
        return params
space = {
    'batch_size': hp.choice('batch_size', np.arange(16, 256, 16, dtype=int)),
    'lr_max': hp.loguniform('lr_max', np.log(1e-5), np.log(1e-2)),
    'weight_decay': hp.loguniform('weight_decay', np.log(1e-6), np.log(1)),
#     'arch': hp.choice('arch', [models.resnet34, models.resnet50, models.resnet101]),
}
trials = Trials()
best = fmin(objective,
            space=space,
            algo=tpe.suggest,
            max_evals=50,
            trials=trials)
import pandas as pd
trials_df = pd.DataFrame(trials.results)
trials_df.to_csv('trials-stage2.csv', sep='\t', index=False)
# trials_df = pd.read_csv('trials-stage1.csv', sep='\t')
# trials_df.sort_values(by='status')
trials_df.sort_values(by='acc', ascending=False)
# trials_df['arch_name'] = trials_df.arch.map(lambda a: a.__name__)
# trials_df.to_csv('trials.csv', sep='\t', index=False)
import seaborn as sns
sns.set(style="whitegrid")
sns.__version__
df = trials_df[trials_df.acc > 0.85]
plt.scatter(1 - df.loss.values, df.acc.values)
fig, ax = plt.subplots()
data = [trials_df[trials_df.status == 'fail'].batch_size.values,
        trials_df[trials_df.status == 'ok'].batch_size.values]
ax.boxplot(data);
fig, ax = plt.subplots()
df = trials_df
df = df[(df.status == 'ok')&(df.batch_size < 100)&(df.batch_size > 20)]
ax.scatter(df.batch_size.values, df.acc.values)
df = trials_df.copy()
df['arch_name'] = df.arch.map(lambda a: a.__name__)
df = df[df.acc > 0.8]
ax = sns.boxplot(x=df.arch_name, y=df.acc)
df = trials_df.copy()
df['acc_loss'] = df.acc * (1 - df.loss)
df = df[df.acc > 0.95]
# df = df[df.arch_name == 'resnet101']
df.sort_values(by='acc_loss', ascending=False)
plt.subplot()
plt.xscale('log')
plt.yscale('log')
plt.minorticks_on()
plt.scatter(x=df.lr_max.values, y=df.weight_decay.values, s=(df.acc.values - 0.95)*1000)<jupyter_output><empty_output><jupyter_text>TTA Experiments<jupyter_code>tta_probs_all = []
tta_labels_all = []

model.eval()
running_loss, running_corrects = 0.0, 0.0
dataloaders = load_data(batch_size=64, tta=True)
for inputs, labels in dataloaders['valid']:
    print('.', end='')
    inputs = inputs.to(device)
    labels = labels.to(device)
    
    with torch.set_grad_enabled(False):
        ns, ncrops, c, h, w = inputs.size()
        outputs = model(inputs.view(-1, c, h, w))
        outputs_avg = outputs.view(ns, ncrops, -1).mean(1)
        _, preds = torch.max(outputs_avg, 1)
        loss = criterion(outputs_avg, labels)
        
        tta_probs_all.append(outputs_avg.cpu().numpy())
        tta_labels_all.append(labels.cpu().numpy())

        running_loss += loss.item() * ns
        running_corrects += torch.sum(preds == labels.data).cpu().numpy()

tta_probs_all = np.concatenate(tta_probs_all)
tta_labels_all = np.concatenate(tta_labels_all)
        
epoch_loss = running_loss / len(dataloaders['valid'].dataset)
epoch_acc = running_corrects / len(dataloaders['valid'].dataset)
epoch_loss, epoch_acc
# np.concatenate(avg_preds_all),
# np.concatenate(labels_all)
probs_all = []
labels_all = []

model.eval()
running_loss, running_corrects = 0.0, 0.0
dataloaders = load_data(batch_size=64, tta=False)
for inputs, labels in dataloaders['valid']:
    print('.', end='')
    inputs = inputs.to(device)
    labels = labels.to(device)
    
    with torch.set_grad_enabled(False):
        outputs = model(inputs.view(-1, c, h, w))
        _, preds = torch.max(outputs, 1)
        loss = criterion(outputs, labels)
        
        probs_all.append(outputs)
        labels_all.append(labels)

        running_loss += loss.item() * ns
        running_corrects += torch.sum(preds == labels.data).cpu().numpy()

probs_all = np.concatenate(probs_all)
labels_all = np.concatenate(labels_all)
        
epoch_loss = running_loss / len(dataloaders['valid'].dataset)
epoch_acc = running_corrects / len(dataloaders['valid'].dataset)
epoch_loss, epoch_acc
for beta in np.linspace(0, 1, 11):
    comb_probs = beta * tta_probs_all + (1 - beta) * probs_all
    corrects = (comb_probs.argmax(axis=1) == labels_all)
    print(beta, corrects.sum(), corrects.mean())
a = np.asarray(list(zip(tta_probs_all.argmax(axis=1), probs_all.argmax(axis=1), labels_all)))
a[~((a[:,0] == a[:,2]) | (a[:,0] == a[:,2]))]<jupyter_output><empty_output><jupyter_text>Cross Validation Model Training<jupyter_code>from flowers.dataset import *
train_valid_ds = TrainValidListDataset(data_path, 'train', 'valid')
from sklearn.model_selection import KFold
np.random.seed(13)

arch = models.resnet101

cv_results = []
cv = KFold(n_splits=5, shuffle=True)
for i, (train_inds, valid_inds) in enumerate(cv.split(train_valid_ds)):
    datasets = {
        'train': SubsetDataset(train_valid_ds, train_inds, train_transform),
        'valid': SubsetDataset(train_valid_ds, valid_inds, valid_transform_tta)
    }
    print(f'Fold: {i}', len(datasets['train']), len(datasets['valid']))
    batch_size = 48
    num_workers = 4
    dataloaders = {
        'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                             shuffle=True, num_workers=num_workers),
        'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                             shuffle=False, num_workers=num_workers)
    }
    epochs = 15
    lr_max = 3e-3
    weight_decay = 5e-2
    
    model = create_model(arch)
    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
                      lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)
    scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,
                                  batches=len(dataloaders['train']), epochs=epochs, debug=False)                  
    criterion = nn.CrossEntropyLoss()
    model, best_loss_head, best_acc_head = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                                      epochs=epochs)
    
    batch_size = 64
    num_workers = 4
    dataloaders = {
        'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,
                                             shuffle=True, num_workers=num_workers),
        'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,
                                             shuffle=False, num_workers=num_workers)
    }
    epochs = 15
    lr_max = 7e-5
    weight_decay = 2e-4
    
    unfreeze(model)
    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),
                      lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)
    scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,
                                  batches=len(dataloaders['train']), epochs=epochs, debug=False)
    criterion = nn.CrossEntropyLoss()
    model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,
                                            epochs=epochs)
    
    model_path = f'resnet-cv-{i}.pth'
    save_model(model, model_path)
    
    d = {
        'model_path': model_path,
        'best_loss_head': best_loss_head,
        'best_acc_head': best_acc_head,
        'best_loss': best_loss,
        'best_acc': best_acc
    }
    cv_results.append(d)
    print(f'Results: {d}')
import pandas as pd
cv_results_df = pd.DataFrame(cv_results)
cv_results_df
# test_ds = torchvision.datasets.ImageFolder(root=str(data_path / 'test'), transform=valid_transform_tta)
# test_dl = torch.utils.data.DataLoader(test_ds, batch_size=64, shuffle=False, num_workers=4)<jupyter_output><empty_output><jupyter_text>Predict and Submit Test Set<jupyter_code>n = 818
for i in range(20):
    print(i, (n - i) / n)
from flowers.test_predict import *
test_ds = torchvision.datasets.ImageFolder(root=str(data_path / 'test'), transform=valid_transform_tta)
test_dl = torch.utils.data.DataLoader(test_ds, batch_size=64, shuffle=False, num_workers=4)
test_preds = predict_test(test_dl, model)
# test_preds = ensemble_predict_test(models.resnet101, test_dl, [f'resnet-cv-{i}.pth' for i in range(5)])
idx_to_class = {v: k for k, v in dataloaders['valid'].dataset.class_to_idx.items()}
create_submission(test_preds, idx_to_class, fn='submission.csv')
# !ls -lah |grep submission
# !kaggle competitions submit oxford-102-flower-pytorch -f submission.csv\
#  -m "resnet101, 15 epochs, bs 48/64, 1cycle + other fastai tricks, lr=3e-3/7e-5, AdamW wd=5e-2/2e-4, basic affine aug, 45 rot, 10xTTA, retrained"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13172259932596003, 'fraction_numerical': 0.018607300517396876, 'mean_word_length': 3.0198435413089104, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '242892', 'license': 'Apache-2.0', 'n_tokens_mistral': 7334, 'n_tokens_neox': 7022, 'n_words': 1239, 'path': 'train_pytorch.ipynb', 'repo_name': 'intsco/udacity-fb-pytorch-project', 'size': '21067'}