<jupyter_start><jupyter_text>1. Batch normalization (2 points)Please make use of the model code from Homework 5 question 2 as you work on this question.1. If your model didn't use batch normalization, add it. If it already had batch normalization, remove it. How does the performance change? Please experiment with different learning rate values, since batch normalization can allow a different range of learning rates to work. You are welcome to write your own batch norm implementaiton or use a pre-existing one from mxnet or pytorch.1. Plot the squared L2 norm of gradients with respect to parameters over the course of training with and without batch normalization. To do this, you will need to compute the gradient of the loss over some data with respect to each parameter, square these gradients, and sum the result across all parameters. You can compute this value at each training step, or after each epoch. What difference does batch normalization make in terms of the squared L2 norm of gradients?  ResNet Without Batch Normalization<jupyter_code>## Train CIFAR10 with PyTorch using ResNet.
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn

import torchvision
import torchvision.transforms as transforms

import os
import argparse
#from utils import progress_bar

import multiprocessing as mp
from torch import linalg as LA
# ResNet Model without batch-norm

class BasicBlock_without_batchnorm(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock_without_batchnorm, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.conv1(x))
        #out = F.relu(self.bn1(self.conv1(x)))
        #out = self.bn2(self.conv2(out))
        out = self.conv2(out)
        out += self.shortcut(x)
        out = F.relu(out)
        return out



class ResNet_without_batchnorm(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet_without_batchnorm, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        #out = F.relu(self.bn1(self.conv1(x)))
        out = F.relu(self.conv1(x))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18_without_batchnorm():
    return ResNet_without_batchnorm(BasicBlock_without_batchnorm, [2, 2, 2, 2])

# Set up trainning environment 
mp.set_start_method("fork")

#parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
#parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
#parser.add_argument('--resume', '-r', action='store_true',
                    #help='resume from checkpoint')
#args = parser.parse_args()

device = 'cuda' if torch.cuda.is_available() else 'cpu'
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
losses_nobatchnorm = [] # losses every time for final learning curve
sum_of_grad_list = []
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(
    trainset, batch_size=128, shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(
    testset, batch_size=100, shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck')
# set up the model
print('==> Building model..')
net = ResNet18_without_batchnorm()
net = net.to(device)
if device == 'cuda':
    net = torch.nn.DataParallel(net)
    cudnn.benchmark = True

#if args.resume:
    # Load checkpoint.
    #print('==> Resuming from checkpoint..')
    #assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    #checkpoint = torch.load('./checkpoint/ckpt.pth')
    #net.load_state_dict(checkpoint['net'])
    #best_acc = checkpoint['acc']
    #start_epoch = checkpoint['epoch']

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.1,
                      momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)
# Training
def train(epoch):
    print('\nEpoch: %d' % epoch)
    net.train()
    train_loss = 0
    correct = 0
    total = 0
    global losses_nobatchnorm
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        sum_grad=0
        inputs, targets = inputs.to(device), targets.to(device)
        optimizer.zero_grad()
        outputs = net(inputs)
        loss = criterion(outputs, targets)
        loss.backward()
        
        # calculate the sum of grad for each epoch
        if batch_idx == 390:
          if isinstance(net, torch.nn.DataParallel):
            net2 = net.module
            for name, param in net2.named_parameters():
              sum_grad = sum_grad + (torch.norm(param,2))**2
            print("squared L2 norm of gradients: %.3f" %sum_grad)
            sum_of_grad_list.append(sum_grad)
        optimizer.step()
        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()
        if batch_idx == 390:
          losses_nobatchnorm.append(train_loss/(batch_idx+1))
          print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
          % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
          #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     #% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))


def test(epoch):
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            if batch_idx == 99:
              print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
              % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
              #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         #% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt.pth')
        best_acc = acc
for epoch in range(start_epoch, start_epoch+20):
    train(epoch)
    test(epoch)
    scheduler.step()
import matplotlib.pyplot as plt
list_plot = [i.detach().cpu().numpy() for i in sum_of_grad_list]
plt.plot(list_plot)
plt.xlabel('epoch')
plt.ylabel('squared L2 norm of gradients')
plt.title('squared L2 norm of gradients vs. No. of epochs')
plt.show()
import matplotlib.pyplot as plt
plt.plot(losses_nobatchnorm)
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Loss vs. No. of epochs')
plt.show()<jupyter_output><empty_output><jupyter_text>Regular ResNet With Batch Normalization<jupyter_code># ResNet Model

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out



class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18():
    return ResNet(BasicBlock, [2, 2, 2, 2])
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(
    trainset, batch_size=128, shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(
    testset, batch_size=100, shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck')
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
losses = [] # losses every time for final learning curve
sum_of_grad_list_reg = [] 
# set up the model
print('==> Building model..')
net_reg = ResNet18()
net_reg = net_reg.to(device)
if device == 'cuda':
    net_reg = torch.nn.DataParallel(net_reg)
    cudnn.benchmark = True

#if args.resume:
    # Load checkpoint.
    #print('==> Resuming from checkpoint..')
    #assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    #checkpoint = torch.load('./checkpoint/ckpt.pth')
    #net.load_state_dict(checkpoint['net'])
    #best_acc = checkpoint['acc']
    #start_epoch = checkpoint['epoch']

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net_reg.parameters(), lr=0.1,
                      momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)
# Training
def train_reg(epoch):
    print('\nEpoch: %d' % epoch)
    net_reg.train()
    train_loss = 0
    correct = 0
    total = 0
    global losses
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        sum_grad=0
        inputs, targets = inputs.to(device), targets.to(device)
        optimizer.zero_grad()
        outputs = net_reg(inputs)
        loss = criterion(outputs, targets)
        loss.backward()
        if batch_idx == 390:
          if isinstance(net, torch.nn.DataParallel):
            net2 = net_reg.module
            for name, param in net2.named_parameters():
              sum_grad = sum_grad + (param.norm(2))**2
            print("squared L2 norm of gradients: %.3f" %sum_grad)
            sum_of_grad_list_reg.append(sum_grad)

        optimizer.step()
        
        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()
        if batch_idx == 390:
          losses.append(train_loss/(batch_idx+1))
          print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
          % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
          #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     #% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))


def test_reg(epoch):
    global best_acc
    net_reg.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net_reg(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            if batch_idx == 99:
              print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
              % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
              #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         #% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net_reg.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt_reg.pth')
        best_acc = acc
for epoch in range(start_epoch, start_epoch+20):
    train_reg(epoch)
    test_reg(epoch)
    scheduler.step()
import matplotlib.pyplot as plt
list_plot = [i.detach().cpu().numpy() for i in sum_of_grad_list_reg]
plt.plot(list_plot)
plt.xlabel('epoch')
plt.ylabel('squared L2 norm of gradients')
plt.title('squared L2 norm of gradients vs. No. of epochs')
plt.show()
import matplotlib.pyplot as plt
plt.plot(losses)
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Loss vs. No. of epochs')
plt.show()
import matplotlib.pyplot as plt
list_plot1 = [i.detach().cpu().numpy() for i in sum_of_grad_list_reg]
list_plot2 = [i.detach().cpu().numpy() for i in sum_of_grad_list]
plt.plot(list_plot1, label="with Batch Norm")
plt.plot(list_plot2, label="without Batch Norm")
plt.xlabel('epoch')
plt.ylabel('squared L2 norm of gradients')
plt.title('squared L2 norm of gradients vs. No. of epochs')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Try different learning rates<jupyter_code>best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
losses_lr005 = [] # losses every time for final learning curve
sum_of_grad_list_reg_lr005 = [] 

net_reg_lr005 = ResNet18()
net_reg_lr005 = net_reg_lr005.to(device)
if device == 'cuda':
    net_reg_lr005 = torch.nn.DataParallel(net_reg_lr005)
    cudnn.benchmark = True

optimizer_lr005 = optim.SGD(net_reg_lr005.parameters(), lr=0.05,
                      momentum=0.9, weight_decay=5e-4)
# Training
def train_reg_lr005(epoch):
    print('\nEpoch: %d' % epoch)
    net_reg_lr005.train()
    train_loss = 0
    correct = 0
    total = 0
    global losses_lr005
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        sum_grad=0
        inputs, targets = inputs.to(device), targets.to(device)
        optimizer_lr005.zero_grad()
        outputs = net_reg_lr005(inputs)
        loss = criterion(outputs, targets)
        loss.backward()
        if batch_idx == 390:
          if isinstance(net, torch.nn.DataParallel):
            net2 = net_reg_lr005.module
            for name, param in net2.named_parameters():
              sum_grad = sum_grad + (param.norm(2))**2
            print("squared L2 norm of gradients: %.3f" %sum_grad)
            sum_of_grad_list_reg_lr005.append(sum_grad)

        optimizer_lr005.step()
        
        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()
        if batch_idx == 390:
          losses_lr005.append(train_loss/(batch_idx+1))
          print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
          % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
          #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     #% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))


def test_reg_lr005(epoch):
    global best_acc
    net_reg_lr005.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net_reg_lr005(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            if batch_idx == 99:
              print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
              % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
              #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         #% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net_reg_lr005.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt_reg_lr005.pth')
        best_acc = acc

for epoch in range(start_epoch, start_epoch+20):
    train_reg_lr005(epoch)
    test_reg_lr005(epoch)
    scheduler.step()
import matplotlib.pyplot as plt
plt.plot(losses_lr005)
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Loss vs. No. of epochs')
plt.show()<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>list_plot1 = [i.detach().cpu().numpy() for i in sum_of_grad_list_reg]
list_plot2 = [i.detach().cpu().numpy() for i in sum_of_grad_list]
list_plot3 = [i.detach().cpu().numpy() for i in sum_of_grad_list_reg_lr005]
plt.plot(list_plot1, label="with Batch Norm, lr=0.1")
plt.plot(list_plot2, label="without Batch Norm, lr=0.1")
plt.plot(list_plot3, label="with Batch Norm, lr=0.05")
plt.xlabel('epoch')
plt.ylabel('squared L2 norm of gradients')
plt.title('squared L2 norm of gradients vs. No. of epochs')
plt.legend()
plt.show()
plt.plot(losses, label="with Batch Norm, lr=0.1")
plt.plot(losses_nobatchnorm, label="without Batch Norm, lr=0.1")
plt.plot(losses_lr005, label="with Batch Norm, lr=0.05")
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Training loss vs. No. of epochs')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>ANSWER: 1. From the training loss vs No. of epochs, we find that using batch norm would increase the training speed, since the model using batch norm would use less epochs to reach lower training loss. Moreover, an appropriate learning rate would further improve the model performance. Here, the model using batch norm and with learning rate of 0.05 is the best among three models. Its training loss descent fastest and achieves the highest accuracy in the testing set. 2. From the squared L2 norm of gradients vs. No. of epochs, we can see that using batch normalization can effectively reduce the absolute value of the gradients.  2. Residual connections (2 points)Please make use of the model code from Homework 5 question 2 as you work on this question.If your model didn't use residual connections, add them. If it already had residual connections, remove them. How does the performance change? Please experiment with applying residual connections around different blocks, where a block is a subset of layers (for example, add a residual connection around a single convolution/nonlinearity combination, or a residual connection around two convolution/nonlinearity combinations). Note that you may need to add 1x1 convolutions in the residual path if your block changes the shape (spatial or number of channels) of the input.  ResNet Model without residual connection<jupyter_code># ResNet Model without residual connection

class BasicBlock_without_connection(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock_without_connection, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        #out += self.shortcut(x)
        out = F.relu(out)
        return out



class ResNet_without_connection(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet_without_connection, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18_without_connection():
    return ResNet_without_connection(BasicBlock_without_connection, [2, 2, 2, 2])
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(
    trainset, batch_size=128, shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(
    testset, batch_size=100, shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck')
# set up the model
print('==> Building model..')
net_noconnect = ResNet18_without_connection()
net_noconnect = net_noconnect.to(device)
if device == 'cuda':
    net_noconnect = torch.nn.DataParallel(net_noconnect)
    cudnn.benchmark = True

#if args.resume:
    # Load checkpoint.
    #print('==> Resuming from checkpoint..')
    #assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    #checkpoint = torch.load('./checkpoint/ckpt.pth')
    #net.load_state_dict(checkpoint['net'])
    #best_acc = checkpoint['acc']
    #start_epoch = checkpoint['epoch']

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net_noconnect.parameters(), lr=0.1,
                      momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
losses_noconnect = [] # losses every time for final learning curve
# Training
def train_noconnect(epoch):
    print('\nEpoch: %d' % epoch)
    net_noconnect.train()
    train_loss = 0
    correct = 0
    total = 0
    global losses_noconnect
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        inputs, targets = inputs.to(device), targets.to(device)
        optimizer.zero_grad()
        outputs = net_noconnect(inputs)
        loss = criterion(outputs, targets)
        loss.backward()
        optimizer.step()
        
        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()
        if batch_idx == 390:
          losses_noconnect.append(train_loss/(batch_idx+1))
          print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
          % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
          #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     #% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))


def test_noconnect(epoch):
    global best_acc
    net_noconnect.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net_noconnect(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            if batch_idx == 99:
              print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
              % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
              #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         #% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net_noconnect.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt_withoutresid.pth')
        best_acc = acc
for epoch in range(start_epoch, start_epoch+20):
    train_noconnect(epoch)
    test_noconnect(epoch)
    scheduler.step()
import matplotlib.pyplot as plt
plt.plot(losses_noconnect)
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Loss vs. No. of epochs')
plt.show()<jupyter_output><empty_output><jupyter_text>Try different connection blocks<jupyter_code># ResNet Model with connection on every layer

class BasicBlock_everyone(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock_everyone, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        
        self.shortcut1 = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut1 = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

        self.shortcut2 = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut2 = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = self.bn1(self.conv1(x))
        out += self.shortcut1(x)
        out = F.relu(out)

        out = self.bn2(self.conv2(out))
        out += self.shortcut2(x)
        out = F.relu(out)
        return out



class ResNet_everyone(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet_everyone, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18_everyone():
    return ResNet_everyone(BasicBlock_everyone, [2, 2, 2, 2])
# Set up trainning environment 
#mp.set_start_method("fork")

#parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
#parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
#parser.add_argument('--resume', '-r', action='store_true',
                    #help='resume from checkpoint')
#args = parser.parse_args()

device = 'cuda' if torch.cuda.is_available() else 'cpu'
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch
losses_everyone = [] # losses every time for final learning curve
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True, download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(
    trainset, batch_size=128, shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(
    root='./data', train=False, download=True, transform=transform_test)
testloader = torch.utils.data.DataLoader(
    testset, batch_size=100, shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck')
# set up the model
print('==> Building model..')
net_everyone = ResNet18_everyone()
net_everyone = net_everyone.to(device)
if device == 'cuda':
    net_everyone = torch.nn.DataParallel(net_everyone)
    cudnn.benchmark = True

#if args.resume:
    # Load checkpoint.
    #print('==> Resuming from checkpoint..')
    #assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    #checkpoint = torch.load('./checkpoint/ckpt.pth')
    #net.load_state_dict(checkpoint['net'])
    #best_acc = checkpoint['acc']
    #start_epoch = checkpoint['epoch']

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net_everyone.parameters(), lr=0.1,
                      momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)
# Training
def train_everyone(epoch):
    print('\nEpoch: %d' % epoch)
    net_everyone.train()
    train_loss = 0
    correct = 0
    total = 0
    global losses_everyone
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        inputs, targets = inputs.to(device), targets.to(device)
        optimizer.zero_grad()
        outputs = net_everyone(inputs)
        loss = criterion(outputs, targets)
        loss.backward()
        optimizer.step()
        
        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()
        if batch_idx == 390:
          losses_everyone.append(train_loss/(batch_idx+1))
          print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
          % (train_loss/(batch_idx+1), 100.*correct/total, correct, total))
          #progress_bar(batch_idx, len(trainloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     #% (train_loss/(batch_idx+1), 100.*correct/total, correct, total))


def test_everyone(epoch):
    global best_acc
    net_everyone.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net_everyone(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

            if batch_idx == 99:
              print('Loss: %.3f | Acc: %.3f%% (%d/%d)'
              % (test_loss/(batch_idx+1), 100.*correct/total, correct, total))
              #progress_bar(batch_idx, len(testloader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                         #% (test_loss/(batch_idx+1), 100.*correct/total, correct, total))

    # Save checkpoint.
    acc = 100.*correct/total
    if acc > best_acc:
        print('Saving..')
        state = {
            'net': net_everyone.state_dict(),
            'acc': acc,
            'epoch': epoch,
        }
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/ckpt_everyone.pth')
        best_acc = acc
for epoch in range(start_epoch, start_epoch+20):
    train_everyone(epoch)
    test_everyone(epoch)
    scheduler.step()
import matplotlib.pyplot as plt
plt.plot(losses_everyone)
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Loss vs. No. of epochs')
plt.show()<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>plt.plot(losses, label="with Residual Connections over every other layer")
plt.plot(losses_everyone, label="with Residual Connections over every layer")
plt.plot(losses_noconnect, label="without Residual Connections")
plt.xlabel('epoch')
plt.ylabel('Training loss')
plt.title('Training loss vs. No. of epochs')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13176592171554063, 'fraction_numerical': 0.028781811135512207, 'mean_word_length': 2.8978851963746224, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '352614', 'license': 'MIT', 'n_tokens_mistral': 13389, 'n_tokens_neox': 12552, 'n_words': 2868, 'path': 'Homework 6-Hua.ipynb', 'repo_name': 'huahanqi/COMP-590-790-DeepLearning-Assignment', 'size': '38705'}