<jupyter_start><jupyter_text>Set Seed for Reproducibility<jupyter_code>def set_seed(seed):
    torch.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED']=str(seed)
    
set_seed(72)<jupyter_output><empty_output><jupyter_text>Preprocess and Load Data<jupyter_code>preprocess = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.4914, 0.4822, 0.4465), (.2023, .1994, .2010)),  #CIFAR10
#      transforms.Normalize((0.507, 0.487, 0.441), (0.267, 0.256, 0.276))      #CIFAR100
#      Cutout(n_holes=1, length=16),               # CutOut
])

train_transform = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomCrop(32, padding=4, fill=128),
#     CIFAR10Policy(),                    # AutoAugment
    preprocess

])

# train_transform.transforms.insert(0, RandAugment(1, 5))                    #RandAugment

test_transform = preprocess

# test_transform = transforms.Compose([
#     transforms.ToTensor(),
#      transforms.Normalize((0.4914, 0.4822, 0.4465), (.2023, .1994, .2010)),  #CIFAR10
#      transforms.Normalize((0.507, 0.487, 0.441), (0.267, 0.256, 0.276))      #CIFAR100
# ])
batch_size = 4

train_data = datasets.CIFAR10(root="./data", train=True, download=True, transform=train_transform)

test_data= datasets.CIFAR10(root="./data", train=False, download=True, transform=test_transform)

# load training data in batches
train_loader = torch.utils.data.DataLoader(
#                       AugMixDataset(train_data, preprocess, no_jsd=True),      # Augmix
                      train_data,
                      batch_size=batch_size,
                      num_workers=8,
                      shuffle=True, 
                      pin_memory=True
                      )

# load test data in batches
test_loader = torch.utils.data.DataLoader(test_data,
                      batch_size=batch_size,
                      num_workers=8,
                      shuffle=False,
                      pin_memory=True
                      )

classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
print(f'Length of train loader is {len(train_loader)}')
print(f'Length of test loader is {len(test_loader)}')<jupyter_output><empty_output><jupyter_text>Build Model<jupyter_code>model = CustomModel()
num_feature = 192*4*4
num_classes = len(classes)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model = torch.nn.DataParallel(model)
fc = Full_layer(num_feature=num_feature, num_classes=len(classes)).to(device)
fc = torch.nn.DataParallel(fc)
best_top1 = 0 # train from start
start_epoch = 0
num_epochs = 50
combine_ratio = 0.5
isda_criterion = ISDALoss(num_feature, num_classes)
ce_criterion = CrossEntropyLoss(smooth_eps=0.1).to(device)
optimizer = Lookahead(DeepMemory([{'params': model.parameters()},
                             {'params': fc.parameters()}], len_memory=len(train_data.data)//batch_size))          # rectified adam wtih lookahead
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_data.data)//batch_size) 
print('Number of model parameters: {}'.format(
        sum([p.data.nelement() for p in model.parameters()]) +
        sum([p.data.nelement() for p in fc.parameters()])
        ))<jupyter_output><empty_output><jupyter_text>Load Checkpoints to resume training<jupyter_code># checkpoint = torch.load('./checkpoint/CustomModel_standard_dmla_isda_ckpt.pth')

# model.module.load_state_dict(checkpoint['model_state_dict'], strict=False)
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
# fc.module.load_state_dict(checkpoint['fc'])
# start_epoch = checkpoint['epoch']
# best_loss = checkpoint['loss']
# best_top1 = checkpoint['top1']   # resume training
# best_top5 = checkpoint['top5']  
# print(f'Top-1 Acc: {best_top1}\t Top-5 Acc: {best_top5}\t Best loss: {best_loss:.4f}\t Best epoch: {start_epoch}')
# print(f'Loaded checkpoint with \n {best_top1}% Top-1 Accuracy, {best_top5}% Top-5 Accuracy, after training for {start_epoch} epochs.')<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>%%time
def train(train_loader, model, fc, criterion, optimizer, epoch):
    print('Training model...\n')
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()
    
    ratio = combine_ratio * (epoch / (num_epochs))
    
    # switch to train mode
    model.train()
    fc.train()

    end = time.time()
    
    for i, (input, target) in enumerate(train_loader):
                
        # measure data loading time
        data_time.update(time.time() - end)
        
        target = target.to(device)
        input_var = Variable(input)
        target_var = Variable(target)
        
        optimizer.zero_grad()
        
        # compute output
        loss, output = criterion(model, fc, input_var, target_var, ratio)
                  
        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.data.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))
       
        loss.backward()
        optimizer.step()
        
        lr_scheduler.step()
        
        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % 1500 == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, top1=top1, top5=top5))
    
    print(' * Acc@1 {top1.avg:.3f} Acc@1 Error {top1_err:.3f}\n'
              ' * Acc@5 {top5.avg:.3f} Acc@5 Error {top5_err:.3f}'
              .format(top1=top1, top1_err=100-top1.avg, top5=top5, top5_err=100-top5.avg))<jupyter_output><empty_output><jupyter_text>Accuracy on test data<jupyter_code>%%time
def validate(test_loader, model, fc, criterion, epoch):
    print('Evaluating model on test data...\n')
    
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()
    
    # switch to evaluate mode
    model.eval()
    fc.eval()
        
    end = time.time()
    for i, (input, target) in enumerate(test_loader):
        target = target.to(device)
        input_var = Variable(input)
        target_var = Variable(target)

        # compute output
        with torch.no_grad():
            features = model(input_var)
            output = fc(features)
            
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target.data, topk=(1, 5))
        losses.update(loss.data.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        top5.update(prec5.item(), input.size(0))

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % 250 == 0:
            print('Test: [{0}/{1}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                   i, len(test_loader), batch_time=batch_time, loss=losses,
                   top1=top1, top5=top5))



    print(' * Acc@1 {top1.avg:.3f} Acc@1 Error {top1_err:.3f}\n'
          ' * Acc@5 {top5.avg:.3f} Acc@5 Error {top5_err:.3f}'
          .format(top1=top1, top1_err=100-top1.avg, top5=top5, top5_err=100-top5.avg))

    return top1, top5, losses<jupyter_output><empty_output><jupyter_text>Checkpoint Model<jupyter_code>%%time
for epoch in range(start_epoch, num_epochs):
    train(train_loader, model, fc, isda_criterion, optimizer, epoch)
    top1, top5, losses = validate(test_loader, model, fc, ce_criterion, epoch)

    if top1.avg > best_top1:
        print('Saving checkpoint')
        state = {
            'model_state_dict': model.module.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'fc': fc.module.state_dict(),
            'epoch': epoch,
            'loss': losses.avg, 
            'top1': top1.avg,
            'top5': top5.avg}
        if not os.path.isdir('checkpoint'):
            os.mkdir('checkpoint')
        torch.save(state, './checkpoint/CustomModel_standard_dmla_isda_ckpt.pth')

        best_top1 = top1.avg<jupyter_output><empty_output><jupyter_text>Accuracy for each class<jupyter_code># class_correct = list(0. for i in range(100))
# class_total = list(0. for i in range(100))
# with torch.no_grad():
#     for data in test_loader:
#         images, labels = data
#         outputs = model(images)
#         _, predicted = torch.max(outputs, 1)
#         c = (predicted == labels).squeeze()
#         for i in range(4):
#             label = labels[i]
#             class_correct[label] += c[i].item()
#             class_total[label] += 1


# for i in range(10):
#     print('Accuracy of %5s : %2d %%' % (
#         classes[i], 100 * class_correct[i] / class_total[i]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1432305279665447, 'fraction_numerical': 0.03366440146366963, 'mean_word_length': 2.6991492652745555, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '211378', 'license': 'MIT', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3246, 'n_words': 592, 'path': 'isda_sample_notebook.ipynb', 'repo_name': 'RichardScottOZ/sota-data-augmentation-and-optimizers', 'size': '9565'}