<jupyter_start><jupyter_text><jupyter_code>!nvidia-smi
!pip install torchinfo
import torchinfo
import os

import torch
import torchvision
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from torch import nn
from torch.utils.data import Dataset, DataLoader 

device = "cuda" if torch.cuda.is_available() else "cpu"
device<jupyter_output><empty_output><jupyter_text>Data loadingLet's import CIFAR10 to build a ResNet50 model on to fit.<jupyter_code>BATCH_SIZE = 32
NUM_WORKERS = os.cpu_count()

# Setup transform to preprocess images
transform = transforms.Compose(
    [transforms.ToTensor(),
     # Normalize to 0.5 mean and 0.5 std across all 3 colour channels of images
     transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))])

# Load training data
train_data = datasets.CIFAR10(root="./data", train=True, download=True, transform=transform)
train_dataloader = DataLoader(train_data, batch_size=BATCH_SIZE,
                              # num workers defines how many CPU cores to use to load data
                              # usually more is better... though you may jam your machine if you use too much...
                              shuffle=True, num_workers=NUM_WORKERS)

# Load testing data
test_data = datasets.CIFAR10(root="./data", train=False, download=True, transform=transform)
test_dataloader = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)
class_names = train_data.classes
class_names
import random
import matplotlib.pyplot as plt
random_idx = random.sample(list(range(len(train_data))), k=1)
print(random_idx)
image, label = train_data[random_idx[0]]
print(image.shape)
image = image.permute(1, 2, 0) / 2 + 0.5 # change image axis order (3, 32, 32) -> (32, 32, 3) & unormalize
print(image.shape)
plt.imshow(image)
plt.title(class_names[label])
plt.axis(False);<jupyter_output>[15593]
torch.Size([3, 32, 32])
torch.Size([32, 32, 3])<jupyter_text>Build the modelWe're going to be replicating the ResNet50 model from the classic "Deep Residual Learning for Image Recognition" paper: https://arxiv.org/pdf/1512.03385.pdfGood implementation here: https://gist.github.com/nikogamulin/7774e0e3988305a78fd73e1c4364aded<jupyter_code>class ResNetBlock(nn.Module):
  def __init__(self, in_features, out_features, stride_size=2, is_identity=True):
    super().__init__()
    self.is_identity = is_identity
    self.expansion = 4 # how much to multiply the output channel by (e.g. 64 -> 256, 128 -> 512)
    self.conv1 = nn.Conv2d(in_channels=in_features, 
                           out_channels=out_features,
                           kernel_size=1,
                           stride=1,
                           padding=1)

    self.batch_norm1 = nn.BatchNorm2d(out_features)

    self.conv2 = nn.Conv2d(in_channels=out_features, 
                            out_channels=out_features, 
                            kernel_size=3,
                            stride=2, # middle (3x3 has the increased stride)
                            padding=1)

    self.batch_norm2 = nn.BatchNorm2d(out_features)

    # Increase out features for last layer in block
    increased_out_features = out_features * self.expansion

    self.conv3 = nn.Conv2d(in_channels=out_features,
                           out_channels=increased_out_features,
                           kernel_size=1,
                           stride=1,
                           padding=0)

    self.batch_norm3 = nn.BatchNorm2d(increased_out_features)

    self.relu = nn.ReLU(inplace=True)

    self.identity_downsample = nn.Sequential(nn.Conv2d(in_channels=out_features,
                                                       out_channels=increased_out_features,
                                                       kernel_size=1,
                                                       stride=2,
                                                       padding=1),
                                             nn.BatchNorm2d(increased_out_features))

  def forward(self, x):
    identity = x
    print(identity.shape)
    print(identity.shape)
    print(self.is_identity)

    # Block 1
    out = self.conv1(x)
    out = self.batch_norm1(out)
    out = self.relu(out)

    # Block 2
    out = self.conv2(out)
    out = self.batch_norm2(out)
    out = self.relu(out)

    # Block 3
    out = self.conv3(out)
    out = self.batch_norm3(out)
    print(f"Out shape: {out.shape}")

    if self.is_identity:
      print("Is identity... downsampling...")
      identity = self.identity_downsample(identity)
      print(f"Identity shape: {identity.shape}")

    out += identity
    out = self.relu(out)

    return out
# Construct the ResNet50 model
model = nn.Sequential()
model.add_module("Conv1", nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3))
model.add_module("BatchNorm1", nn.BatchNorm2d(num_features=64))
model.add_module("ReLU1", nn.ReLU(inplace=True))
model.add_module("MaxPool1", nn.MaxPool2d(kernel_size=3, stride=2, padding=1))
previous_feature_size = 64
for feature_size in [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3:
  print(f"Feature size: {feature_size}")
  if previous_feature_size == feature_size:
    stride_size = 1
    is_identity = False
  else:
    stride_size = 2
    is_identity = True
  print(f"Stride size: {stride_size}")
  print(f"Is identity? {is_identity}")
  model.add_module(f"{feature_size}_block", 
                   ResNetBlock(in_features=feature_size, 
                               out_features=feature_size, 
                               stride_size=stride_size,
                               is_identity=True))
  previous_feature_size = feature_size
model.add_module("avg_pool", nn.AdaptiveAvgPool2d((1, 1)))
model.add_module("fully_connected", nn.Linear(in_features=previous_feature_size, out_features=len(class_names)))
model.to(device)
model(torch.unsqueeze(train_data[0][0].to(device), dim=0))
from torchinfo import summary
summary(model, input_size=(BATCH_SIZE, 3, 224, 224))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1027635280489823, 'fraction_numerical': 0.03210325997021347, 'mean_word_length': 2.737786023500309, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '57713', 'license': 'MIT', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1879, 'n_words': 401, 'path': 'resnet50_twitch.ipynb', 'repo_name': 'mrdbourke/pytorch-resnet-twitch', 'size': '6043'}