<jupyter_start><jupyter_text>**Base resnet18 on CIFAR100:**Accuracy = 48.99%Robustness = 0.03%**Supervised MTL resnet18 on CIFAR100:**Accuracy = 50.47%Robustness = 25.64%  Natural Base Model resnet18  Train base resnet-18 model with same hyper parameter  Data Loader<jupyter_code>import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms 
import torchvision.models as models
import tqdm
from torchvision.datasets import CIFAR100

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Hyper-parameters
num_epochs = 100
learning_rate = 0.0003
BATCH_SIZE = 256
weight_decay = 1e-4
PATH = '/content/drive/MyDrive/resnet18'

device


# Image preprocessing modules
transform = transforms.Compose([
    transforms.Pad(4),
    transforms.RandomHorizontalFlip(),
    transforms.RandomCrop(32),
    transforms.ToTensor()])

# CIFAR-100 dataset
train_dataset = torchvision.datasets.CIFAR100(root='../../data/',
                                             train=True, 
                                             transform=transform,
                                             download=True)

test_dataset = torchvision.datasets.CIFAR100(root='../../data/',
                                            train=False, 
                                            transform=transforms.ToTensor())

# Data loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                           batch_size=BATCH_SIZE, 
                                           shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
                                          batch_size=BATCH_SIZE, 
                                          shuffle=False)


device<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>model = models.resnet18(pretrained=False)
model = model.to(device)

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), learning_rate, weight_decay=weight_decay)

scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader), eta_min=0,
                                                           last_epoch=-1)

# For updating learning rate

# Train the model
total_step = len(train_loader)
curr_lr = learning_rate
for epoch in range(num_epochs):
  for i, (images, labels) in enumerate(train_loader):
    images = images.to(device)
    labels = labels.to(device)

    # Forward pass
    outputs = model(images)
    loss = criterion(outputs, labels)

    # Backward and optimize
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if (i+1) % 100 == 0:
      print ("Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}"
                .format(epoch+1, num_epochs, i+1, total_step, loss.item()))

  # Decay learning rate
  if epoch >= 10:
    scheduler.step()                                                           

  if epoch %10 == 0:
    checkpoint_name = 'checkpoint_resnet18-base-cifar100_{:04d}.pth.tar'.format(epoch)
    torch.save(model.state_dict(), PATH+'/'+checkpoint_name)

checkpoint_name = 'checkpoint_resnet18-base-cifar100_{:04d}.pth.tar'.format(num_epochs)
torch.save(model.state_dict(), PATH+'/'+checkpoint_name)<jupyter_output><empty_output><jupyter_text>Load the model and Attack  Load the model<jupyter_code>PATH = '/content/drive/MyDrive/resnet18/checkpoint_resnet18-base-cifar100_0100.pth.tar'
model_state_dict = torch.load(PATH)
model = models.resnet18(pretrained=False)
model.load_state_dict(model_state_dict)
model = model.to(device)<jupyter_output><empty_output><jupyter_text>PGD attack the model and save images  FGSM<jupyter_code>!pip install torchattacks
import os
import sys
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset

import torchvision.utils
from torchvision import models
import torchvision.datasets as dsets
import torchvision.transforms as transforms

import torchattacks
from torchattacks import PGD, FGSM

# from models import Holdout, Target
# from utils import imshow
batch_size = 24

cifar100_train = dsets.CIFAR100(root='./data', train=True,
                              download=True, transform=transforms.ToTensor())
cifar100_test  = dsets.CIFAR100(root='./data', train=False,
                              download=True, transform=transforms.ToTensor())

train_loader = torch.utils.data.DataLoader(cifar100_train,
                                           batch_size=batch_size,
                                           shuffle=True)
test_loader = torch.utils.data.DataLoader(cifar100_test,
                                          batch_size=batch_size,
                                          shuffle=False)
atk = FGSM(model, eps=8/255)
# atk.set_return_type('int') # Save as integer.
atk.save(data_loader=test_loader, save_path="/content/drive/MyDrive/mtlsimclr/cifar100_fgsm.pt", verbose=True)<jupyter_output>- Save progress: 100.00 % / Robust accuracy: 2.77 % / L2: 1.71249 (0.012 it/s) 	
- Save complete!<jupyter_text>PGD-7<jupyter_code>atk = PGD(model, eps=8/255, alpha=2/255, steps=7)
# atk.set_return_type('int') # Save as integer.
atk.save(data_loader=test_loader, save_path="/content/drive/MyDrive/mtlsimclr/cifar100_pgd.pt", verbose=True)<jupyter_output>- Save progress: 100.00 % / Robust accuracy: 0.37 % / L2: 1.33795 (0.059 it/s) 	
- Save complete!<jupyter_text>PGD-20<jupyter_code>atk = PGD(model, eps=8/255, alpha=2/255, steps=20)
# atk.set_return_type('int') # Save as integer.
atk.save(data_loader=test_loader, save_path="/content/drive/MyDrive/mtlsimclr/cifar100_pgd_20.pt", verbose=True)<jupyter_output>- Save progress: 100.00 % / Robust accuracy: 0.18 % / L2: 1.46321 (0.162 it/s) 	
- Save complete!<jupyter_text>PGD-50<jupyter_code>atk = PGD(model, eps=8/255, alpha=2/255, steps=50)
# atk.set_return_type('int') # Save as integer.
atk.save(data_loader=test_loader, save_path="/content/drive/MyDrive/mtlsimclr/cifar100_pgd_50.pt", verbose=True)<jupyter_output>- Save progress: 100.00 % / Robust accuracy: 0.13 % / L2: 1.50696 (0.399 it/s) 	
- Save complete!<jupyter_text>Test the model  Natural Accuracy<jupyter_code># Test the model
model.eval()
with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in test_loader:
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))

# Save the model checkpoint
torch.save(model.state_dict(), 'resnet.ckpt')<jupyter_output>Accuracy of the model on the test images: 48.99 %<jupyter_text>Robust Accuracy  FGSM<jupyter_code>adv_images, adv_labels = torch.load("/content/drive/MyDrive/mtlsimclr/cifar100_fgsm.pt")
adv_data = TensorDataset(adv_images, adv_labels)
adv_loader_fgsm = DataLoader(adv_data, batch_size=128, shuffle=False)
model.eval()

correct = 0
total = 0

for images, labels in adv_loader_fgsm:
    
    images = images.cuda()
    outputs = model(images)
    
    _, predicted = torch.max(outputs.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 2.77 %<jupyter_text>PGD-7<jupyter_code>adv_images, adv_labels = torch.load("/content/drive/MyDrive/mtlsimclr/cifar100_pgd.pt")
adv_data = TensorDataset(adv_images, adv_labels)
adv_loader_7 = DataLoader(adv_data, batch_size=128, shuffle=False)
model.eval()

correct = 0
total = 0

for images, labels in adv_loader_7:
    
    images = images.cuda()
    outputs = model(images)
    
    _, predicted = torch.max(outputs.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 0.37 %<jupyter_text>PGD-20<jupyter_code>adv_images, adv_labels = torch.load("/content/drive/MyDrive/mtlsimclr/cifar100_pgd_20.pt")
adv_data = TensorDataset(adv_images, adv_labels)
adv_loader_20 = DataLoader(adv_data, batch_size=128, shuffle=False)
model.eval()

correct = 0
total = 0

for images, labels in adv_loader_20:
    
    images = images.cuda()
    outputs = model(images)
    
    _, predicted = torch.max(outputs.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 0.18 %<jupyter_text>PGD-50<jupyter_code>adv_images, adv_labels = torch.load("/content/drive/MyDrive/mtlsimclr/cifar100_pgd_50.pt")
adv_data = TensorDataset(adv_images, adv_labels)
adv_loader_50 = DataLoader(adv_data, batch_size=128, shuffle=False)
model.eval()

correct = 0
total = 0

for images, labels in adv_loader_50:
    
    images = images.cuda()
    outputs = model(images)
    
    _, predicted = torch.max(outputs.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 0.13 %<jupyter_text>Sup MTL Resnet18  Load the model<jupyter_code>PATH = '/content/drive/MyDrive/resnet18/checkpoint_resnet18-sup-mtl_0100.pth.tar'
model_state_dict = torch.load(PATH)

class ResNetTwoPrediction(nn.Module):
  def __init__(self):
    super(ResNetTwoPrediction, self).__init__()
    self.backbone = models.resnet18(pretrained=False)
    dim = self.backbone.fc.in_features
    # print(dim)
    self.backbone.fc = nn.Identity() # place holder for the fc layer
    self.backbone.fine = nn.Sequential(nn.Linear(dim, dim), 
                                       nn.ReLU(), 
                                       nn.Linear(in_features=512, out_features=100, bias=True))
    self.backbone.coarse = nn.Sequential(nn.Linear(dim, dim), 
                                         nn.ReLU(), 
                                         nn.Linear(in_features=512, out_features=20, bias=True))

  def forward(self, x):
    base_representation = self.backbone(x)
    fine_out = self.backbone.fine(base_representation)
    coarse_out = self.backbone.coarse(base_representation)
    return fine_out, coarse_out

model = ResNetTwoPrediction()
model.load_state_dict(model_state_dict)
model = model.to(device)<jupyter_output><empty_output><jupyter_text>Natural Accuracy<jupyter_code># Test the model
model.eval()
with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in test_loader:
        images = images.to(device)
        labels = labels.to(device)
        fine_out, coarse_out = model(images)
        _, predicted = torch.max(fine_out.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    print('Accuracy of the model on the test images: {} %'.format(100 * correct / total))

# Save the model checkpoint
torch.save(model.state_dict(), 'resnet.ckpt')<jupyter_output>Accuracy of the model on the test images: 50.47 %<jupyter_text>Robust Accuracy  FGSM<jupyter_code>model.eval()

correct = 0
total = 0

for images, labels in adv_loader_fgsm:
    images = images.to(device)
    labels = labels.to(device)
    
    fine_out, coarse_out = model(images)
    _, predicted = torch.max(fine_out.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 23.20 %<jupyter_text>PGD-7<jupyter_code>model.eval()

correct = 0
total = 0

for images, labels in adv_loader_7:
    images = images.to(device)
    labels = labels.to(device)
    
    fine_out, coarse_out = model(images)
    _, predicted = torch.max(fine_out.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 25.64 %<jupyter_text>PGD-20<jupyter_code>model.eval()

correct = 0
total = 0

for images, labels in adv_loader_20:
    images = images.to(device)
    labels = labels.to(device)
    
    fine_out, coarse_out = model(images)
    _, predicted = torch.max(fine_out.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 23.59 %<jupyter_text>PGD-50<jupyter_code>model.eval()

correct = 0
total = 0

for images, labels in adv_loader_50:
    images = images.to(device)
    labels = labels.to(device)
    
    fine_out, coarse_out = model(images)
    _, predicted = torch.max(fine_out.data, 1)
    
    total += labels.size(0)
    correct += (predicted == labels.cuda()).sum()
    
print('Robust accuracy: %.2f %%' % (100 * float(correct) / total))<jupyter_output>Robust accuracy: 23.49 %
{'fraction_non_alphanumeric': 0.12298973809159136, 'fraction_numerical': 0.033925562873334356, 'mean_word_length': 3.2690421706440014, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 39, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '588091', 'license': 'MIT', 'n_tokens_mistral': 4742, 'n_tokens_neox': 4343, 'n_words': 849, 'path': 'cifar100_test.ipynb', 'repo_name': 'ClaireZixiWang/SimCLR', 'size': '13058'}