<jupyter_start><jupyter_text>Training on all Data<jupyter_code>!curl --header 'Host: doc-04-b8-docs.googleusercontent.com' --user-agent 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' --header 'Accept-Language: en-CA,en-US;q=0.7,en;q=0.3' --referer 'https://drive.google.com/drive/u/0/recent' --cookie 'AUTH_ke38de5om415qdl9mh10ijp9ovprni2j_nonce=oa9soihsi4uus' --header 'Upgrade-Insecure-Requests: 1' 'https://doc-04-b8-docs.googleusercontent.com/docs/securesc/2n3g1c92fvum2ti2873eohe2s5i599l8/4jijecna13ujp5q6ss2frm01a5hhgh2e/1590592350000/11397432307160908430/11397432307160908430/12O1GZmLO1t_iyhlHEDSKBL9PhZF8nMps?e=download&h=04295518330385903249&authuser=0&nonce=oa9soihsi4uus&user=11397432307160908430&hash=k8kobi0tptkkjo6mksffoh9ijrk0m3hb' --output 'veggie_fruit_data.zip'

!unzip veggie_fruit_data.zip
import torch                                                                                                                                                                                                
import torchvision
import torchvision.transforms as transforms
from torch.utils import data
import argparse
import torch.nn as nn
import tqdm
import torch.optim as optim
from sklearn.metrics import precision_recall_fscore_support

def train():
    data_dir = 'veggie_fruit_data/'
    total_classes = 64
    val_every_nepochs = 1 
    batch_size = 30
    nepochs = 80
    num_workers = 8
    train_input_folder = data_dir + 'train/'
    validate_input_folder = data_dir + 'validate/'

    # Training Data Loader
    train_transform = transforms.Compose([
       transforms.RandomHorizontalFlip(),
       transforms.RandomVerticalFlip(),
       transforms.RandomAffine(scale=(0.9, 1.25), shear=0.2, degrees=0),
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])  
    train_dataset = torchvision.datasets.ImageFolder(train_input_folder,
                                                     transform=train_transform)
    train_loader = data.DataLoader(train_dataset, batch_size=batch_size,
                                   shuffle=True, num_workers=num_workers)

    # Validation Data Loader
    val_transform = transforms.Compose([
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])
    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                     transform=val_transform)
    val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                   shuffle=False, num_workers=num_workers)

    # Create Model
    model = torchvision.models.vgg19(pretrained=True)
    model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
    model = model.cuda()
    criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()

    # Create Optimizer and freeze backbone
    for param in model.features.parameters():
        param.requires_grad = False                                                                                                                                                                         
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # Training/Val Loop
    model.train()
    for epoch in range(nepochs):
        print('Epoch ', epoch)
        tqdm_train = tqdm.tqdm(train_loader)
        for image, label in tqdm_train:
            image = image.cuda()
            label = label.cuda()

            optimizer.zero_grad()
            pred = model(image)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()
        if epoch % val_every_nepochs == 0:
            model.eval()
            with torch.no_grad():
                correct = 0.0
                tqdm_val = tqdm.tqdm(val_loader)
                metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                           'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                           'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}

                for image, label in tqdm_val:
                    image = image.cuda()
                    pred = model(image)
                    pred_labels = pred.detach().cpu().argmax(dim=1)
                    for key in metrics.keys():
                        prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                                  average=key)
                        metrics[key]['prec'] += prec
                        metrics[key]['rec'] += recall
                        metrics[key]['fscore'] += fscore
                    correct += (pred_labels == label).sum().item()
                print('Accuracy ', correct/(len(tqdm_val)*batch_size))
                for key in metrics.keys():
                    print(' Metric %s: prec rec fscore'%key)
                    out_str = ''
                    for k2 in metrics[key].keys():
                        metrics[key][k2] /= (len(tqdm_val))
                        out_str += str(metrics[key][k2]) + ' '
                    print(out_str)
                torch.save(model.state_dict(), 'ckpt.pth')


train()
                                             
import torch
import torchvision
from sklearn.metrics import precision_recall_fscore_support

data_dir = 'veggie_fruit_data/'
total_classes = 64
batch_size = 30
num_workers = 8
train_input_folder = data_dir + 'train/'
validate_input_folder = data_dir + 'validate/'

# Validation Data Loader
val_transform = transforms.Compose([
    transforms.Resize((224,224)),
    transforms.ToTensor(),
    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                  transform=val_transform)
val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                shuffle=False, num_workers=num_workers)
# Create Model
model = torchvision.models.vgg19(pretrained=True)
model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
model = model.cuda()

model.load_state_dict(torch.load("ckpt.pth"))

model.eval()
with torch.no_grad():
    correct = 0.0
    tqdm_val = tqdm.tqdm(val_loader)
    metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}
    labels = None
    preds = None
    count_cls = {}
    for image, label in tqdm_val:
        image = image.cuda()
        pred = model(image)
        pred_labels = pred.detach().cpu().argmax(dim=1)
        if labels is None:
          labels = label
          preds = pred_labels
        else:
          labels = torch.cat((labels, label), dim=0)
          preds = torch.cat((preds, pred_labels), dim=0)

        for key in metrics.keys():
            prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                      average=key)
            metrics[key]['prec'] += prec
            metrics[key]['rec'] += recall
            metrics[key]['fscore'] += fscore
        correct += (pred_labels == label).sum().item()
    
    print('Accuracy ', correct/(len(tqdm_val)*batch_size))
    for key in metrics.keys():
        print(' Metric %s: prec rec fscore'%key)
        out_str = ''
        for k2 in metrics[key].keys():
            metrics[key][k2] /= (len(tqdm_val))
            out_str += str(metrics[key][k2]) + ' '
        print(out_str)

    print('Fscore per class ') 
    prec, recall, fscore, sprt = precision_recall_fscore_support(labels, preds, average=None)
    for i in range(len(fscore)):
      print('Class : ', i, 'Fscore : ', fscore[i], ' Support: ', sprt[i])<jupyter_output>0%|          | 0/14 [00:00<?, ?it/s][A[A[A[A[A[A[A[A/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))








  7%|▋         | 1/14 [00:24<05:13, 24.12s/it][A[A[A[A[A[A[A[A







 14%|█▍        | 2/14 [00:24<03:24, 17.00s/it][A[A[A[A[A[A[A[A/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))








 21%|██▏       | 3/14 [00:24<02:12, 12.02s/it][A[A[A[A[A[A[A[A







 29%|██▊       | 4/14 [00:25<01:25,  8.53s/it][A[A[A[A[A[A[A[A



[...]<jupyter_text>This is what he got```        precision    recall  f1-score   support        artichoke       0.80      1.00      0.89         8          avocado       0.97      0.97      0.97        35baby_head_lettuce       0.00      0.00      0.00         4           banana       0.96      1.00      0.98        68            beets       0.73      1.00      0.84         8     bitter_melon       0.96      1.00      0.98        52         bok_choy       1.00      0.70      0.82        23         broccoli       0.91      0.99      0.95        88        butternut       1.00      0.67      0.80         3          cabbage       1.00      1.00      1.00        60       cantaloupe       1.00      1.00      1.00         6           carrot       1.00      1.00      1.00        16      cauliflower       0.97      1.00      0.99        70           celery       0.92      0.67      0.77        18    cherry_tomato       1.00      0.88      0.93         8          collard       1.00      0.50      0.67         2             corn       0.94      0.96      0.95        47  curly_leaf_kale       0.00      0.00      0.00         2           daikon       1.00      1.00      1.00         7 dandelion_greens       0.00      0.00      0.00         2     dragon_fruit       0.98      1.00      0.99        56           durian       0.95      1.00      0.98        20         eggplant       0.98      0.98      0.98        65     english_peas       0.80      1.00      0.89         4       fava_beans       0.00      0.00      0.00         3           ginger       1.00      0.92      0.96        12       green_bean       0.97      1.00      0.98        30      green_onion       0.89      0.97      0.93       108   kabocha_squash       0.94      1.00      0.97        50             kale       0.78      1.00      0.88        14    korean_radish       1.00      0.33      0.50         6    lacinato_kale       1.00      0.50      0.67         2            leeks       1.00      0.25      0.40         4            lemon       0.97      1.00      0.99        39       lemongrass       1.00      0.79      0.88        14             lime       0.93      0.96      0.95        28        long_bean       1.00      0.76      0.86        21         mandarin       1.00      0.85      0.92        13     napa_cabbage       1.00      0.50      0.67         4             okra       0.88      0.92      0.90        24            onion       1.00      1.00      1.00        12           papaya       0.91      0.96      0.93        72           peanut       0.93      1.00      0.96        13        pineapple       1.00      0.98      0.99        42      pomegranate       0.79      0.94      0.86        16           potato       1.00      1.00      1.00         4    rainbow_chard       1.00      1.00      1.00         1      red_chicory       1.00      0.50      0.67         2        red_onion       0.98      1.00      0.99        54     red_radishes       1.00      0.75      0.86         4      redbor_kale       0.00      0.00      0.00         2    savoy_cabbage       1.00      1.00      1.00         2       strawberry       1.00      0.92      0.96        12  sugar_snap_peas       0.00      0.00      0.00         2    summer_squash       0.62      0.83      0.71         6           tomato       0.99      0.97      0.98        92      water_melon       1.00      1.00      1.00        46  white_nectarine       0.75      0.90      0.82        10      white_peach       1.00      0.40      0.57         5     yellow_peach       0.60      1.00      0.75         3         accuracy                           0.95      1444        macro avg       0.85      0.79      0.80      1444     weighted avg       0.94      0.95      0.94      1444```<jupyter_code>!curl --header 'Host: doc-00-b8-docs.googleusercontent.com' --user-agent 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' --header 'Accept-Language: en-CA,en-US;q=0.7,en;q=0.3' --referer 'https://drive.google.com/drive/u/0/recent' --cookie 'AUTH_ke38de5om415qdl9mh10ijp9ovprni2j=11397432307160908430|1590678675000|1ao7uvnvq5htv3cn4uh5h7t1c6h19t7n' --header 'Upgrade-Insecure-Requests: 1' 'https://doc-00-b8-docs.googleusercontent.com/docs/securesc/2n3g1c92fvum2ti2873eohe2s5i599l8/nv73h5juji466jedc5qgbe4fide32th7/1590678750000/11397432307160908430/11397432307160908430/1H66JjnNpiGX0q7s0HJ3xUNkFILPQEG1_?h=04295518330385903249&e=download&authuser=0' --output 'new_veggie_fruit_data.zip'
!unzip new_veggie_fruit_data.zip<jupyter_output>% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 2020M    0 2020M    0     0  74.3M      0 --:--:--  0:00:27 --:--:-- 71.6M
Archive:  new_veggie_fruit_data.zip
   creating: new_veggie_fruit_data/
   creating: new_veggie_fruit_data/base/
   creating: new_veggie_fruit_data/base/train/
   creating: new_veggie_fruit_data/base/train/water_melon/
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_1207.JPG  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_03343.jpg  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_03335.jpg  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_1186.JPG  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_1204.JPG  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_03340.jpg  
  inflating: new_veggie_fruit_data/base/train/water_melon/IMG_03337.jpg  
  inflating: new_veggie_fruit[...]<jupyter_text>Base Training on Base Classes<jupyter_code>import torch                                                                                                                                                                                                
import torchvision
import torchvision.transforms as transforms
from torch.utils import data
import argparse
import torch.nn as nn
import tqdm
import torch.optim as optim
from sklearn.metrics import precision_recall_fscore_support

def train():
    data_dir = 'new_veggie_fruit_data/base/'
    total_classes = 44
    val_every_nepochs = 1 
    batch_size = 30
    nepochs = 80
    num_workers = 8
    train_input_folder = data_dir + 'train/'
    validate_input_folder = data_dir + 'validate/'

    # Training Data Loader
    train_transform = transforms.Compose([
       transforms.RandomHorizontalFlip(),
       transforms.RandomVerticalFlip(),
       transforms.RandomAffine(scale=(0.9, 1.25), shear=0.2, degrees=0),
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])  
    train_dataset = torchvision.datasets.ImageFolder(train_input_folder,
                                                     transform=train_transform)
    train_loader = data.DataLoader(train_dataset, batch_size=batch_size,
                                   shuffle=True, num_workers=num_workers)

    # Validation Data Loader
    val_transform = transforms.Compose([
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])
    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                     transform=val_transform)
    val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                   shuffle=False, num_workers=num_workers)

    # Create Model
    model = torchvision.models.vgg19(pretrained=True)
    model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
    model = model.cuda()
    criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()

    # Create Optimizer and freeze backbone
    for param in model.features.parameters():
        param.requires_grad = False                                                                                                                                                                         
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # Training/Val Loop
    model.train()
    for epoch in range(nepochs):
        print('Epoch ', epoch)
        tqdm_train = tqdm.tqdm(train_loader)
        for image, label in tqdm_train:
            image = image.cuda()
            label = label.cuda()

            optimizer.zero_grad()
            pred = model(image)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()
        if epoch % val_every_nepochs == 0:
            model.eval()
            with torch.no_grad():
                correct = 0.0
                tqdm_val = tqdm.tqdm(val_loader)
                metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                           'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                           'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}

                for image, label in tqdm_val:
                    image = image.cuda()
                    pred = model(image)
                    pred_labels = pred.detach().cpu().argmax(dim=1)
                    for key in metrics.keys():
                        prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                                  average=key)
                        metrics[key]['prec'] += prec
                        metrics[key]['rec'] += recall
                        metrics[key]['fscore'] += fscore
                    correct += (pred_labels == label).sum().item()
                print('Accuracy ', correct/(len(tqdm_val)*batch_size))
                for key in metrics.keys():
                    print(' Metric %s: prec rec fscore'%key)
                    out_str = ''
                    for k2 in metrics[key].keys():
                        metrics[key][k2] /= (len(tqdm_val))
                        out_str += str(metrics[key][k2]) + ' '
                    print(out_str)
                torch.save(model.state_dict(), 'base_ckpt.pth')


train()
                                             
import torch
import torchvision
from sklearn.metrics import precision_recall_fscore_support

data_dir = 'new_veggie_fruit_data/base/'
total_classes = 44
batch_size = 30
num_workers = 8
train_input_folder = data_dir + 'train/'
validate_input_folder = data_dir + 'validate/'

# Validation Data Loader
val_transform = transforms.Compose([
    transforms.Resize((224,224)),
    transforms.ToTensor(),
    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                  transform=val_transform)
val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                shuffle=False, num_workers=num_workers)
# Create Model
model = torchvision.models.vgg19(pretrained=True)
model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
model = model.cuda()

model.load_state_dict(torch.load("base_ckpt.pth"))

model.eval()
with torch.no_grad():
    correct = 0.0
    tqdm_val = tqdm.tqdm(val_loader)
    metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}
    labels = None
    preds = None
    count_cls = {}
    for image, label in tqdm_val:
        image = image.cuda()
        pred = model(image)
        pred_labels = pred.detach().cpu().argmax(dim=1)
        if labels is None:
          labels = label
          preds = pred_labels
        else:
          labels = torch.cat((labels, label), dim=0)
          preds = torch.cat((preds, pred_labels), dim=0)

        for key in metrics.keys():
            prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                      average=key)
            metrics[key]['prec'] += prec
            metrics[key]['rec'] += recall
            metrics[key]['fscore'] += fscore
        correct += (pred_labels == label).sum().item()
    
    print('Accuracy ', correct/(len(tqdm_val)*batch_size))
    for key in metrics.keys():
        print(' Metric %s: prec rec fscore'%key)
        out_str = ''
        for k2 in metrics[key].keys():
            metrics[key][k2] /= (len(tqdm_val))
            out_str += str(metrics[key][k2]) + ' '
        print(out_str)

    print('Fscore per class ') 
    prec, recall, fscore, sprt = precision_recall_fscore_support(labels, preds, average=None)
    for i in range(len(fscore)):
      print('Class : ', i, 'Fscore : ', fscore[i], ' Support: ', sprt[i])<jupyter_output>31%|███       | 4/13 [00:21<01:06,  7.36s/it]/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
 46%|████▌     | 6/13 [00:24<00:29,  4.27s/it]/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
100%|██████████| 13/13 [00:38<00:00,  2.93s/it]<jupyter_text>Finetune on Fewshot (Base+Novel) Data<jupyter_code>!curl --header 'Host: doc-0g-b8-docs.googleusercontent.com' --user-agent 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' --header 'Accept-Language: en-CA,en-US;q=0.7,en;q=0.3' --referer 'https://drive.google.com/drive/u/0/recent' --cookie 'AUTH_ke38de5om415qdl9mh10ijp9ovprni2j_nonce=c5n597cbmsdpc' --header 'Upgrade-Insecure-Requests: 1' 'https://doc-0g-b8-docs.googleusercontent.com/docs/securesc/2n3g1c92fvum2ti2873eohe2s5i599l8/l0acbnvntupol3vuvovcrhbuna712ccf/1590771825000/11397432307160908430/11397432307160908430/1INdAM0bNL_02QrY40D784ECrJXaGpBvD?e=download&h=04295518330385903249&authuser=0&nonce=c5n597cbmsdpc&user=11397432307160908430&hash=7aost88t7st6dn21qu5cn9rc51sgiacq' --output 'base_ckpt.pth'
#from google.colab import drive
#drive.mount('/content/drive')
#!cp drive/My\ Drive/veggie_fruit_data.zip .
#!curl --header 'Host: doc-00-b8-docs.googleusercontent.com' --user-agent 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0' --header 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' --header 'Accept-Language: en-CA,en-US;q=0.7,en;q=0.3' --referer 'https://drive.google.com/drive/u/0/recent' --cookie 'AUTH_ke38de5om415qdl9mh10ijp9ovprni2j_nonce=ke2thums9bnve' --header 'Upgrade-Insecure-Requests: 1' 'https://doc-00-b8-docs.googleusercontent.com/docs/securesc/2n3g1c92fvum2ti2873eohe2s5i599l8/2o7ikpo16uhblm2sn6d7c7gifoe0q5cn/1590788625000/11397432307160908430/11397432307160908430/1H66JjnNpiGX0q7s0HJ3xUNkFILPQEG1_?e=download&h=04295518330385903249&authuser=0&nonce=ke2thums9bnve&user=11397432307160908430&hash=9pp2d3i2i7s6p8pv0hd6qh87d5icj4p7' --output 'new_veggie_fruit_data.zip'
!unzip new_veggie_fruit_data.zip
#!unzip veggie_fruit_data.zip
#!ls
import torch                                                                                                                                                                                                
import torchvision
import torchvision.transforms as transforms
from torch.utils import data
import argparse
import torch.nn as nn
import tqdm
import torch.optim as optim
from sklearn.metrics import precision_recall_fscore_support

def my_load_dict(model, base_pth, nbase):
  base_state_dict = torch.load(base_pth)
  model_dict = model.state_dict()
  # 1. filter out unnecessary keys
  base_state_dict = {k: v for k, v in base_state_dict.items() if k in model_dict}
  for k, v in base_state_dict.items():
      if 'classifier.6' in k:
          base_state_dict[k] = torch.cat((base_state_dict[k], model_dict[k][nbase:]))
  # 2. overwrite entries in the existing state dict
  model_dict.update(base_state_dict)
  model.load_state_dict(model_dict)

def train():
    data_dir = 'new_veggie_fruit_data/fewshot/'
    total_classes = 64
    val_every_nepochs = 1 
    batch_size = 30
    nepochs = 50
    num_workers = 8
    train_input_folder = data_dir + 'train/'
    validate_input_folder = data_dir + 'validate/'

    # Training Data Loader
    train_transform = transforms.Compose([
       transforms.RandomHorizontalFlip(),
       transforms.RandomVerticalFlip(),
       transforms.RandomAffine(scale=(0.9, 1.25), shear=0.2, degrees=0),
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])  
    target_transform = lambda lbl: lbl+ 44
    train_dataset = torchvision.datasets.ImageFolder(train_input_folder,
                                                     transform=train_transform,
                                                     target_transform=target_transform)
    train_loader = data.DataLoader(train_dataset, batch_size=batch_size,
                                   shuffle=True, num_workers=num_workers)

    # Validation Data Loader
    val_transform = transforms.Compose([
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])
    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                   transform=val_transform,
                                                   target_transform=target_transform)
    val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                   shuffle=False, num_workers=num_workers)

    # Create Model
    model = torchvision.models.vgg19(pretrained=True)
    model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
    model = model.cuda()
    my_load_dict(model, 'base_ckpt.pth', 44)

    criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()

    # Create Optimizer and freeze backbone
    for param in model.features.parameters():
        param.requires_grad = False                                                                                                                                                                         
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # Training/Val Loop
    model.train()
    for epoch in range(nepochs):
        print('Epoch ', epoch)
        tqdm_train = tqdm.tqdm(train_loader)
        for image, label in tqdm_train:
            image = image.cuda()
            label = label.cuda()

            optimizer.zero_grad()
            pred = model(image)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()
        if epoch % val_every_nepochs == 0:
            model.eval()
            with torch.no_grad():
                correct = 0.0
                tqdm_val = tqdm.tqdm(val_loader)
                metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                           'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                           'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}

                for image, label in tqdm_val:
                    image = image.cuda()
                    pred = model(image)
                    pred_labels = pred.detach().cpu().argmax(dim=1)
                    for key in metrics.keys():
                        prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                                  average=key)
                        metrics[key]['prec'] += prec
                        metrics[key]['rec'] += recall
                        metrics[key]['fscore'] += fscore
                    correct += (pred_labels == label).sum().item()
                print('Accuracy ', correct/(len(tqdm_val)*batch_size))
                for key in metrics.keys():
                    print(' Metric %s: prec rec fscore'%key)
                    out_str = ''
                    for k2 in metrics[key].keys():
                        metrics[key][k2] /= (len(tqdm_val))
                        out_str += str(metrics[key][k2]) + ' '
                    print(out_str)
                torch.save(model.state_dict(), 'fewshot_ckpt.pth')


train()
                                             
import torch
import torchvision
from sklearn.metrics import precision_recall_fscore_support

def validate(class_set='base'):
  total_classes = 64
  batch_size = 30
  num_workers = 8
  
  # Validation Data Loader
  val_transform = transforms.Compose([
      transforms.Resize((224,224)),
      transforms.ToTensor(),
      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
  ])
  if class_set == 'base':
    data_dir = 'new_veggie_fruit_data/base/'
    validate_input_folder = data_dir + 'validate/'

    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                    transform=val_transform)
  else:
    data_dir = 'new_veggie_fruit_data/fewshot/'
    validate_input_folder = data_dir + 'validate/'

    target_transform = lambda lbl: lbl + 44 
    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                    transform=val_transform,
                                                  target_transform=target_transform)
    
  val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                  shuffle=False, num_workers=num_workers)
  # Create Model
  model = torchvision.models.vgg19(pretrained=True)
  model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
  model = model.cuda()

  model.load_state_dict(torch.load("fewshot_ckpt.pth"))

  model.eval()
  with torch.no_grad():
      correct = 0.0
      tqdm_val = tqdm.tqdm(val_loader)
      metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                  'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                  'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}
      labels = None
      preds = None
      count_cls = {}
      for image, label in tqdm_val:
          image = image.cuda()
          pred = model(image)
          pred_labels = pred.detach().cpu().argmax(dim=1)
          if labels is None:
            labels = label
            preds = pred_labels
          else:
            labels = torch.cat((labels, label), dim=0)
            preds = torch.cat((preds, pred_labels), dim=0)

          for key in metrics.keys():
              prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                        average=key)
              metrics[key]['prec'] += prec
              metrics[key]['rec'] += recall
              metrics[key]['fscore'] += fscore
          correct += (pred_labels == label).sum().item()
      
      print('Accuracy ', correct/(len(tqdm_val)*batch_size))
      for key in metrics.keys():
          print(' Metric %s: prec rec fscore'%key)
          out_str = ''
          for k2 in metrics[key].keys():
              metrics[key][k2] /= (len(tqdm_val))
              out_str += str(metrics[key][k2]) + ' '
          print(out_str)

      print('Fscore per class ') 
      prec, recall, fscore, sprt = precision_recall_fscore_support(labels, preds, average=None)
      for i in range(len(fscore)):
        print('Class : ', i, 'Fscore : ', fscore[i], ' Support: ', sprt[i])

print('Evaluation on Base Classes ')
validate(class_set='base')

print('Evaluation on fewshot classes')
validate(class_set='fewshot')<jupyter_output>Evaluation on Base Classes<jupyter_text>Base Training and Finetuning using CosSim Model<jupyter_code>import torch                                                                                                                                                                                                
import torchvision
import torchvision.transforms as transforms
from torch.utils import data
import argparse
import torch.nn as nn
import tqdm
import torch.optim as optim
from sklearn.metrics import precision_recall_fscore_support

class CosSimVGG19(nn.Module):
    def __init__(self, num_classes):
        super().__init__()
        self.num_classes = num_classes
        temp_backbone = torchvision.models.vgg19(pretrained=True)
        self.features = temp_backbone.features
        self.backbone_classifier = temp_backbone.classifier[:-1]
        self.cls_score = nn.Linear(4096, num_classes, bias=True)
        self.scale = nn.Parameter(torch.tensor(1.0))

    def forward(self, x):
        x = self.features(x)
        x = self.backbone_classifier(x.view(-1, 512*7*7))

        x_norm = torch.norm(x, p=2, dim=1).unsqueeze(1).expand_as(x)
        x_normalized = x.div(x_norm + 1e-5)

        # normalize weight
        temp_norm = torch.norm(self.cls_score.weight.data,
                               p=2, dim=1).unsqueeze(1).expand_as(
            self.cls_score.weight.data)
        self.cls_score.weight.data = \
                self.cls_score.weight.data.div(temp_norm + 1e-5)

        cos_dist = self.cls_score(x_normalized)
        scores = self.scale * cos_dist
        return scores

def my_load_dict(model, base_pth, nbase):
  base_state_dict = torch.load(base_pth)
  model_dict = model.state_dict()
  # 1. filter out unnecessary keys
  base_state_dict = {k: v for k, v in base_state_dict.items() if k in model_dict}
  for k, v in base_state_dict.items():
      if 'classifier.6' in k:
          base_state_dict[k] = torch.cat((base_state_dict[k], model_dict[k][nbase:]))
  # 2. overwrite entries in the existing state dict
  model_dict.update(base_state_dict)
  model.load_state_dict(model_dict)

def train(class_set='base', model_type='fc'):
    val_every_nepochs = 1 
    batch_size = 30
    nepochs = 50
    num_workers = 8
    
    if class_set == 'base':
      total_classes = 44
      data_dir = 'new_veggie_fruit_data/base/'
      train_input_folder = data_dir + 'train/'
      validate_input_folder = data_dir + 'validate/'
    else:
      total_classes = 64
      data_dir = 'new_veggie_fruit_data/fewshot/'
      train_input_folder = data_dir + 'train/'
      validate_input_folder = data_dir + 'validate/'

    # Training Data Loader
    train_transform = transforms.Compose([
       transforms.RandomHorizontalFlip(),
       transforms.RandomVerticalFlip(),
       transforms.RandomAffine(scale=(0.9, 1.25), shear=0.2, degrees=0),
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])  
    if class_set == 'fewshot':
      target_transform = lambda lbl: lbl+ 44
      train_dataset = torchvision.datasets.ImageFolder(train_input_folder,
                                                       transform=train_transform,
                                                       target_transform=target_transform)
    else:
        train_dataset = torchvision.datasets.ImageFolder(train_input_folder,
                                                       transform=train_transform)
    train_loader = data.DataLoader(train_dataset, batch_size=batch_size,
                                   shuffle=True, num_workers=num_workers)

    # Validation Data Loader
    val_transform = transforms.Compose([
       transforms.Resize((224,224)),
       transforms.ToTensor(),
       transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])

    if class_set == 'fewshot':
      val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                     transform=val_transform,
                                                     target_transform=target_transform)
    else:
        val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                     transform=val_transform)
        
    val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                   shuffle=False, num_workers=num_workers)

    # Create Model
    if model_type == 'cossim':
      model = CosSimVGG19(total_classes)
    else:
      model = torchvision.models.vgg19(pretrained=True)
      model.classifier[6] = nn.Linear(4096, total_classes, bias=True)

    model = model.cuda()

    if class_set == 'fewshot':
      my_load_dict(model, 'base_ckpt.pth', 44)

    criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()

    # Create Optimizer and freeze backbone
    for param in model.features.parameters():
        param.requires_grad = False                                                                                                                                                                         
    optimizer = optim.Adam(model.parameters(), lr=0.001)

    # Training/Val Loop
    model.train()
    for epoch in range(nepochs):
        print('Epoch ', epoch)
        tqdm_train = tqdm.tqdm(train_loader)
        for image, label in tqdm_train:
            image = image.cuda()
            label = label.cuda()

            optimizer.zero_grad()
            pred = model(image)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()
        if epoch % val_every_nepochs == 0:
            model.eval()
            with torch.no_grad():
                correct = 0.0
                tqdm_val = tqdm.tqdm(val_loader)
                metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                           'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                           'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}

                for image, label in tqdm_val:
                    image = image.cuda()
                    pred = model(image)
                    pred_labels = pred.detach().cpu().argmax(dim=1)
                    for key in metrics.keys():
                        prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                                  average=key)
                        metrics[key]['prec'] += prec
                        metrics[key]['rec'] += recall
                        metrics[key]['fscore'] += fscore
                    correct += (pred_labels == label).sum().item()
                print('Accuracy ', correct/(len(tqdm_val)*batch_size))
                for key in metrics.keys():
                    print(' Metric %s: prec rec fscore'%key)
                    out_str = ''
                    for k2 in metrics[key].keys():
                        metrics[key][k2] /= (len(tqdm_val))
                        out_str += str(metrics[key][k2]) + ' '
                    print(out_str)
                torch.save(model.state_dict(), '%s_ckpt.pth'%class_set)
train(class_set='base', model_type='cossim')
import torch
import torchvision
from sklearn.metrics import precision_recall_fscore_support

def validate(class_set='base'):
  total_classes = 64
  batch_size = 30
  num_workers = 8
  
  # Validation Data Loader
  val_transform = transforms.Compose([
      transforms.Resize((224,224)),
      transforms.ToTensor(),
      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
  ])
  if class_set == 'base':
    data_dir = 'new_veggie_fruit_data/base/'
    validate_input_folder = data_dir + 'validate/'

    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                    transform=val_transform)
  else:
    data_dir = 'new_veggie_fruit_data/fewshot/'
    validate_input_folder = data_dir + 'validate/'

    target_transform = lambda lbl: lbl + 44 
    val_dataset = torchvision.datasets.ImageFolder(validate_input_folder,
                                                    transform=val_transform,
                                                  target_transform=target_transform)
    
  val_loader = data.DataLoader(val_dataset, batch_size=batch_size,
                                  shuffle=False, num_workers=num_workers)
  # Create Model
  model = torchvision.models.vgg19(pretrained=True)
  model.classifier[6] = nn.Linear(4096, total_classes, bias=True)
  model = model.cuda()

  model.load_state_dict(torch.load("fewshot_ckpt.pth"))

  model.eval()
  with torch.no_grad():
      correct = 0.0
      tqdm_val = tqdm.tqdm(val_loader)
      metrics = {'macro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},                                                                                                                               
                  'micro': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0},
                  'weighted': {'prec': 0.0, 'rec': 0.0, 'fscore': 0.0}}
      labels = None
      preds = None
      count_cls = {}
      for image, label in tqdm_val:
          image = image.cuda()
          pred = model(image)
          pred_labels = pred.detach().cpu().argmax(dim=1)
          if labels is None:
            labels = label
            preds = pred_labels
          else:
            labels = torch.cat((labels, label), dim=0)
            preds = torch.cat((preds, pred_labels), dim=0)

          for key in metrics.keys():
              prec, recall, fscore, _ = precision_recall_fscore_support(label, pred_labels,
                                                                        average=key)
              metrics[key]['prec'] += prec
              metrics[key]['rec'] += recall
              metrics[key]['fscore'] += fscore
          correct += (pred_labels == label).sum().item()
      
      print('Accuracy ', correct/(len(tqdm_val)*batch_size))
      for key in metrics.keys():
          print(' Metric %s: prec rec fscore'%key)
          out_str = ''
          for k2 in metrics[key].keys():
              metrics[key][k2] /= (len(tqdm_val))
              out_str += str(metrics[key][k2]) + ' '
          print(out_str)

      print('Fscore per class ') 
      prec, recall, fscore, sprt = precision_recall_fscore_support(labels, preds, average=None)
      for i in range(len(fscore)):
        print('Class : ', i, 'Fscore : ', fscore[i], ' Support: ', sprt[i])

print('Evaluation on Base Classes ')
validate(class_set='base')

print('Evaluation on fewshot classes')
validate(class_set='fewshot')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11008784674877618, 'fraction_numerical': 0.0544739253861457, 'mean_word_length': 1.7056546719080738, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 17, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 255, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '651518', 'license': 'MIT', 'n_tokens_mistral': 15059, 'n_tokens_neox': 13483, 'n_words': 2584, 'path': 'SeeingTheWorld_Fewshot.ipynb', 'repo_name': 'MSiam/seeing-the-world-2.0', 'size': '44737'}