<jupyter_start><jupyter_text><jupyter_code>!pip install git+https://github.com/cuongvng/neural-networks-with-PyTorch.git
!nvidia-smi
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import sys
sys.path.append("../..")
from utils.training_helpers import train_cnn
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)<jupyter_output>cuda<jupyter_text>Load CIFAR dataset<jupyter_code>transform = transforms.Compose([
                                transforms.Resize((224,224)),
                                transforms.ToTensor() # convert data from PIL image to tensor
                                ]) 
cifar_train = torchvision.datasets.CIFAR10(root="../data/", train=True, 
                                     transform=transform, 
                                     target_transform=None, 
                                     download=True)
cifar_test = torchvision.datasets.CIFAR10(root="../data/", train=False, 
                                     transform=transform, 
                                     target_transform=None, 
                                     download=True)

batch_size = 128
train_loader = torch.utils.data.DataLoader(
    dataset=cifar_train,
    batch_size=128,
    shuffle=True,
)

test_loader = torch.utils.data.DataLoader(
    dataset=cifar_test,
    batch_size=128,
    shuffle=True,

)
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')<jupyter_output>Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ../data/cifar-10-python.tar.gz<jupyter_text>Let's dive into ResNet's architechture.There are multiple types of ResNet, such as ResNet-18, ResNet-34, ResNet-50, ResNet-152, etc., but they has the same core: the **Residual Block**.In this notebook, I will demonstrate the ResNet-34.         Residual Block<jupyter_code>class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1, use_1x1_conv=False):
        super(ResidualBlock, self).__init__()

        self.use_1x1_conv = use_1x1_conv
        self.conv3x3_1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1, stride=stride)
        self.conv3x3_2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)
        self.conv1x1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride)
        self.bn = nn.BatchNorm2d(num_features=out_channels)
    
    def forward(self, X):
        X_original = X.clone()
        
        X = self.conv3x3_1(X)
        X = F.relu(self.bn(X))
        X = self.conv3x3_2(X)
        X = self.bn(X)
        
        if self.use_1x1_conv:
            X_original = self.conv1x1(X_original)

        return F.relu(X + X_original)<jupyter_output><empty_output><jupyter_text>Let's see the structures of different ResNet variants. ![Structures of ResNets](https://github.com/cuongvng/neural-networks-with-PyTorch/blob/resnet/CNNs/ResNet/img/ResNetStructures.png?raw=1)Source: [the original ResNet paper](https://arxiv.org/pdf/1512.03385.pdf) As we can see, all the variants have the two `7x7 Conv` and `3x3 MaxPooling` layers at the beginning and the two `AvgPooling` and `Dense` Layers at the end. The difference lies in the intermediate layers, which contains multiple stacked Residual Blocks, which are denoted inside big square brackets.Note that we add a BatchNormalization layer after each Convolutional layer.Now let's implement the whole ResNet-34 model.<jupyter_code>class ResNet34(nn.Module):
    def __init__(self):
        super(ResNet34, self).__init__()
        
        # Declare each layer on the table above:
        
        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3),
            nn.BatchNorm2d(num_features=64), 
            nn.ReLU()
        )
        self.conv2_x = nn.Sequential(
            nn.MaxPool2d(kernel_size=3, stride=2, padding=1),
            ResidualBlock(in_channels=64, out_channels=64), 
            ResidualBlock(in_channels=64, out_channels=64),
            ResidualBlock(in_channels=64, out_channels=64), 
        )
        
        # The first Residual Block in each subsequent layer use 1x1 convolutional layer and `stride=2` 
        # to halve the height and width of the input tensor.
        # This does not apply to the above `self.conv2_x` because it already has the `MaxPool2d` layer to do so.
        
        self.conv3_x = nn.Sequential(
            ResidualBlock(in_channels=64, out_channels=128, stride=2, use_1x1_conv=True), 
            ResidualBlock(in_channels=128, out_channels=128), 
            ResidualBlock(in_channels=128, out_channels=128), 
            ResidualBlock(in_channels=128, out_channels=128), 
        )
        self.conv4_x = nn.Sequential(
            ResidualBlock(in_channels=128, out_channels=256, stride=2, use_1x1_conv=True), 
            ResidualBlock(in_channels=256, out_channels=256), 
            ResidualBlock(in_channels=256, out_channels=256), 
            ResidualBlock(in_channels=256, out_channels=256), 
            ResidualBlock(in_channels=256, out_channels=256), 
            ResidualBlock(in_channels=256, out_channels=256), 
        )
        self.conv5_x = nn.Sequential(
            ResidualBlock(in_channels=256, out_channels=512, stride=2, use_1x1_conv=True), 
            ResidualBlock(in_channels=512, out_channels=512), 
            ResidualBlock(in_channels=512, out_channels=512), 
        )
        
        # The Global Average Pooling layer will be replaced by an average pooling function in the `forward` method.
        # The reason why I do not declare it as a torch.nn layer here is that for Global Pooling, the kernel size is the 
        # shape (height, width) of the input tensor, thus it outputs an 1x1 tensor (over the channel dimension).
        # And since I do not know (or just be lazy to calculate) its kernel size, I will print the input shape
        # at the `forward` method to get the kernel size.
        
        # The last dense layer will output 10 classes for the CIFAR dataset, not 1000 for the ImageNet dataset as on the table.
        self.fc = nn.Linear(in_features=512, out_features=10)
        
    def forward(self, X):
        X = X.type(torch.float)
        
        X = self.conv1(X)
        X = self.conv2_x(X)
        X = self.conv3_x(X)
        X = self.conv4_x(X)
        X = self.conv5_x(X)
        
        # Apply global avg pooling 2d (kernel size = (height, width) to get output's spatial dimension = (1,1))
        X = F.avg_pool2d(X, kernel_size=(X.shape[2], X.shape[3]))
        X = torch.flatten(X, start_dim=1)
        X = self.fc(X)
        
        return X
net = ResNet34()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(),lr=0.001)
n_epochs = 10
train_cnn(net, device, train_loader, test_loader, optimizer, criterion, n_epochs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09981489391997722, 'fraction_numerical': 0.03460059803502777, 'mean_word_length': 2.7046413502109705, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '52715', 'license': 'MIT', 'n_tokens_mistral': 2396, 'n_tokens_neox': 2220, 'n_words': 530, 'path': 'CNNs/ResNet/ResNet.ipynb', 'repo_name': 'cuongvng/neural-networks-with-PyTorch', 'size': '7023'}