<jupyter_start><jupyter_text><jupyter_code># Mount Google Drive
from google.colab import drive # import drive from google colab

ROOT = "/content/drive"     # default location for the drive
print(ROOT)                 # print content of ROOT (Optional)

drive.mount(ROOT)           # we mount the google drive at /content/drive
from google.colab import files

uploaded = files.upload()
from Model import Model
try:
    from torchsummary import summary
except ModuleNotFoundError:
    !pip install -q torchsummary
    from torchsummary import summary

try:
    import torch
except ModuleNotFoundError:
    from os import path
    from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag
    platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())

    accelerator = 'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'

    !pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.3.1-{platform}-linux_x86_64.whl
    import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
try:
    import torchvision
except ModuleNotFoundError:
    !pip install -q torchvision

from torchvision import datasets, transforms
cuda = True
train_batch_size = 256
test_batch_size = 124
best_loss = float("inf")
best_epoch = -1
dataset_path = './cifar10'
gsync_save = True
if gsync_save:
    try:
        import utils
    except ModuleNotFoundError:
        !wget https://raw.githubusercontent.com/StefOe/colab-pytorch-utils/HEAD/utils.py
        import utils

    gsync = utils.GDriveSync()
cuda = cuda and torch.cuda.is_available()
trainset = datasets.CIFAR10(root=dataset_path, train=True, download=True)
train_mean = trainset.data.mean(axis=(0,1,2))/255  # [0.49139968  0.48215841  0.44653091]
train_std = trainset.data.std(axis=(0,1,2))/255  # [0.24703223  0.24348513  0.26158784]
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize(train_mean, train_std),
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(train_mean, train_std),
])
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
    root=dataset_path, train=True, download=True,
    transform=transform_train),
    batch_size=train_batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
    datasets.CIFAR10(root=dataset_path, train=False, download=True,
    transform=transform_test),
    batch_size=test_batch_size, shuffle=False, **kwargs)
def train(epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        if cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)

        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % 100 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.data))
def test(epoch, best_loss, best_epoch, lr):
    model.eval()
    test_loss = 0
    correct = 0
    for data, target in test_loader:
        if cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)

        output = model(data)
        # sum up batch loss
        test_loss += criterion(output, target).data
        # get the index of the max log-probability
        pred = output.data.max(1, keepdim=True)[1]
        correct += pred.eq(target.data.view_as(pred)).long().cpu().sum()

    test_loss /= len(test_loader.dataset)
    print(
        '\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
            test_loss, correct, len(test_loader.dataset), 100. * correct /
            len(test_loader.dataset)))
    
    if test_loss < best_loss:
        best_epoch = epoch
        best_loss = test_loss
        torch.save(model, "best_"+str(lr)+".pt")
        try:
          if gsync_save:
              gsync.update_file_to_folder("best_"+str(lr)+".pt")
        except:
          print('Failed to gsync_save.')

        try:
          model_save_name = 'best_'+str(lr)
          path = F"/content/drive/My Drive/dl-reproducibility-project/model/{model_save_name}" 
          torch.save(model.state_dict(), path+'.epoch-{}.pt'.format(epoch))
        except:
          print('Failed to save best model to personal google drive')

    return best_loss, best_epoch, correct
try:
  os.makedirs('/content/drive/My Drive/dl-reproducibility-project/model/')
except:
  print('')
for lr in [0.1,0.05,0.01]:
  print("Training with ", lr)

  
  try: 
    del model
  except:
    print('')
  
  # Maybe it's redundant 
  try:
    del criterion
    del optimizer
    del scheduler
  except:
    print('')
  
  try:
    torch.cuda.empty_cache()
  except:
    print('')
  
  #
  # Chnage Here
  #
  # Choose Model A by basemodel=[True, False, False] and choose B by basemodel=[False, True, False]
  # Choose the modification by modifiedModel=[True, False, False, False]. The first one is the model without any modification.
  # The scecond one is the strided. The third: ConvPool. The last: All CNN.
  model = Model(modifiedModel=[False, False, True, False])
  if cuda:
      model.cuda()

  criterion = nn.CrossEntropyLoss()
  optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)
  scheduler = optim.lr_scheduler.MultiStepLR(
      optimizer, milestones=[200, 250, 300], gamma=0.1)
  print(model)
  summary(model.model, input_size=(3, 32, 32))
  for epoch in range(350):
      scheduler.step()
      train(epoch)
      best_loss, best_epoch, acc = test(epoch, best_loss, best_epoch, lr)
      if (epoch>100)&(acc.item()>0)&((acc.item()/len(test_loader.dataset))<=0.1):
        print("Stop at acc of test set:",acc.item()/len(test_loader.dataset))
        break
  # if in Google Colab, download your model with this
  #from google.colab import files
  #files.download("best_"+str(lr)+"pt")<jupyter_output>Training with  0.1


Model(
  (model): Sequential(
    (0): Dropout(p=0.2, inplace=False)
    (1): Conv2d(3, 96, kernel_size=(5, 5), stride=(1, 1), padding=(1, 1))
    (2): ReLU()
    (3): Conv2d(96, 96, kernel_size=(5, 5), stride=(1, 1), padding=(1, 1))
    (4): ReLU()
    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (6): Dropout(p=0.5, inplace=False)
    (7): Conv2d(96, 192, kernel_size=(5, 5), stride=(1, 1), padding=(1, 1))
    (8): ReLU()
    (9): Conv2d(192, 192, kernel_size=(5, 5), stride=(1, 1), padding=(1, 1))
    (10): ReLU()
    (11): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (12): Dropout(p=0.5, inplace=False)
    (13): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (14): ReLU()
    (15): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1))
    (16): ReLU()
    (17): Conv2d(192, 10, kernel_size=(1, 1), stride=(1, 1))
    (18): ReLU()
    (19): AdaptiveAvgPool2d(output_size=1)
[...]
{'fraction_non_alphanumeric': 0.14240549828178695, 'fraction_numerical': 0.03848797250859107, 'mean_word_length': 3.33611442193087, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '126056', 'license': 'MIT', 'n_tokens_mistral': 2694, 'n_tokens_neox': 2501, 'n_words': 590, 'path': 'modelA/ConvPool-CNN-A.ipynb', 'repo_name': 'wtsyang/dl-reproducibility-project', 'size': '7275'}