<jupyter_start><jupyter_text>Tutorial: How to train your model when you cannot trust onthe annotations? - SIGBRAPI 2020Authors: <NAME>, <NAME>\github: https://github.com/filipe-research/tutorial_noisylabels DATASET: "MNIST ("Modified National Institute of Standards and Technology") is the de facto “hello world” dataset of computer vision. Since its release in 1999, this classic dataset of handwritten images has served as the basis for benchmarking classification algorithms. As new machine learning techniques emerge, MNIST remains a reliable resource for researchers and learners alike."[Read more.](https://www.kaggle.com/c/digit-recognizer)  Setupimport libraries, hyperparameters and main methods<jupyter_code>#@title Import Dependencies

import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.utils.data import Subset
import numpy as np
import random
import matplotlib.pyplot as plt
import torch.nn.functional as F
from sklearn.mixture import GaussianMixture
import seaborn as sn
import pandas as pd
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
#@title Define Hyperparameters

training_size= 2000
num_epochs = 50 # number of times which the entire dataset is passed throughout the model
batch_size = 128 # the size of input data took for one iteration
lr = 1e-3 # size of step
seed=123

random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
#@title Downloading MNIST data

train_data = dsets.MNIST(root = './data', train = True,
                        transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,)),
                       ]), download = True)

test_data = dsets.MNIST(root = './data', train = False,
                       transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,)),
                       ]))
# Reduce training size  (just for fast training during the tutorial)
train_data.data = train_data.data[:training_size]
train_data.targets = train_data.targets[:training_size]
#generate dataloaders
train_loader = torch.utils.data.DataLoader(dataset = train_data,
                                             batch_size = batch_size,
                                             shuffle = True,num_workers=4,pin_memory=True)

eval_loader = torch.utils.data.DataLoader(dataset = train_data,
                                             batch_size = batch_size,
                                             shuffle = False,num_workers=4,pin_memory=True)

test_loader = torch.utils.data.DataLoader(dataset = test_data,
                                      batch_size = batch_size, 
                                      shuffle = False,num_workers=4)
#plot images of dataset

examples = enumerate(test_loader)
batch_idx, (example_data, example_targets) = next(examples)

fig = plt.figure(figsize=(10,6))
for i in range(15):
  plt.subplot(3,5,i+1)
  plt.imshow(example_data[i][0], cmap='gray', interpolation='none')
  plt.title("Ground Truth: {}".format(example_targets[i]))
  plt.xticks([])
  plt.yticks([])
#@title CNN Arquitecture
#CNN Architecture

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 30)
        self.fc2 = nn.Linear(30, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return(x)
ce_loss = nn.CrossEntropyLoss().to(device)
#@title test method
def test(net, loss_function, loader):
  net.eval()
  test_loss = 0
  correct = 0
  
  with torch.no_grad():
    for data, target in loader:
      data, target = data.to(device), target.to(device)
      output = net(data)
      test_loss += loss_function(output, target)
      _, pred = torch.max(output, 1)  
      correct += pred.eq(target).cpu().sum().item() 
  acc = 100.*correct/len(loader.dataset)
  test_loss /= len(loader)
  
  return acc, test_loss
#@title evaluate method

def eval(net, loader):
  net.eval()
  
  losses = torch.zeros(len(loader.dataset))    
  CE = nn.CrossEntropyLoss(reduction='none')
  with torch.no_grad():
    for i ,(images,labels) in enumerate(loader):
        images = images.to(device)
        labels = labels.to(device)
        outputs = net(images)
        loss = CE(outputs, labels)
        for b in range(images.size(0)):
          idx = batch_size*i+b
          losses[idx]=loss[b] 
  
  losses = (losses-losses.min())/(losses.max()-losses.min())    
  input_loss = losses.reshape(-1,1)

  # fit a two-component GMM to the loss
  gmm = GaussianMixture(n_components=2,max_iter=50,tol=1e-2,reg_covar=5e-4)
  gmm.fit(input_loss)
  prob = gmm.predict_proba(input_loss) 
  prob = prob[:,gmm.means_.argmin()] 
  return losses, prob


def train_epoch(net, loss_function, dataloader, optimizer):
  train_loss = train_acc = correct = 0
  net.train()
  for i ,(images,labels) in enumerate(dataloader):
    images = images.to(device)
    labels = labels.to(device)
    
    optimizer.zero_grad()
    outputs = net(images)
    loss = loss_function(outputs, labels)
    train_loss+= loss
    loss.backward()

    optimizer.step()

    _, pred = torch.max(outputs, 1)  
    correct += pred.eq(labels).cpu().sum().item() 
        
  train_loss/=len(dataloader)
  train_acc=100.*correct/len(dataloader.dataset)
  return train_loss, train_acc
def train_test_model(loss_function):
  torch.manual_seed(seed)
  net = Net()
  net = net.to(device)
  optimizer = torch.optim.Adam( net.parameters(), lr=lr) 

  logs={'train_loss':[],'train_acc':[], 'test_loss':[], 'test_acc':[],'eval_loss':[]}

  for epoch in range(num_epochs):
    train_loss, train_acc = train_epoch(net, loss_function, train_loader, optimizer)
    
    logs['train_loss'].append(train_loss)
    logs['train_acc'].append(train_acc)

    eval_loss, _ = eval(net, eval_loader)
    logs['eval_loss'].append(eval_loss)
    
    acc_test, loss_test=test(net, loss_function, test_loader)
    logs['test_loss'].append(loss_test)
    logs['test_acc'].append(acc_test)

    print('Epoch [{}/{}],\ttrain loss:{:.4f},\ttrain acc:{:.2f},\ttest loss:{:.4f},\ttest accuracy:{:.2f}'.format(epoch,num_epochs,train_loss,train_acc, loss_test, acc_test))
  return logs<jupyter_output><empty_output><jupyter_text>Run Baseline<jupyter_code>#run baseline
logs_baseline = train_test_model(ce_loss)
fig = plt.figure(figsize=(14,4))
plt.subplot(1,2,1)
plt.plot(logs_baseline['train_acc'], label='train', linewidth=2)
plt.plot(logs_baseline['test_acc'], label='test', linewidth=2)
plt.legend(frameon=False)
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.ylabel("Accuracy", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');

plt.subplot(1,2,2)
plt.plot(logs_baseline['train_loss'], label='Training loss', linewidth=2)
plt.plot(logs_baseline['test_loss'], label='Validation loss', linewidth=2)
plt.legend(frameon=False)
# plt.grid()
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.ylabel("Loss", labelpad=15, fontsize=12, color="#333533", fontweight='bold');<jupyter_output><empty_output><jupyter_text>Generate Noisy MNISTBuild synthetic noise benchmark on MNIST<jupyter_code>gt_labels = train_data.targets.clone()   #ground truth labels (clean)
##generate noise

transition = {0:0,2:0,4:7,7:7,1:1,9:1,3:5,5:3,6:6,8:8} # class transition for asymmetric noise

def generate_noise(nr, gt_labels, noise_mode='sym'):
  noise_label = []
  idx = list(range(len(gt_labels)))
  random.shuffle(idx)
  num_noise = int(nr*len(gt_labels))            
  noise_idx = idx[:num_noise]
  for i in range(len(gt_labels)):
      if i in noise_idx:
        if noise_mode=='sym':
          noiselabel = random.randint(0,9)  
          noise_label.append(noiselabel)
        elif noise_mode=='asym':   
          noiselabel = transition[gt_labels[i].item()]
          noise_label.append(noiselabel)                    
      else:    
        noise_label.append(gt_labels[i])  
  return torch.tensor(noise_label)
#generate noisy_benchmark
label_noise={}
label_noise['50_sym']=generate_noise(0.5, gt_labels, 'sym')
label_noise['70_sym']=generate_noise(0.7, gt_labels, 'sym')
label_noise['40_asym']=generate_noise(0.4, gt_labels, 'asym')
print('real noise rate:{:.2f}'.format((np.array(label_noise['70_sym'])!=np.array(gt_labels)).sum()/len(gt_labels)))
#plot images of dataset

# train_data.targets = label_noise['70_sym']  #update current labels
train_loader.dataset.targets = label_noise['70_sym'].clone()
examples = enumerate(train_loader)
batch_idx, (example_data, example_targets) = next(examples)

fig = plt.figure(figsize=(10,6))
for i in range(15):
  plt.subplot(3,5,i+1)
  plt.imshow(example_data[i][0], cmap='gray', interpolation='none')
  plt.title("label: {}".format(example_targets[i]))
  plt.xticks([])
  plt.yticks([])
noisy_map = np.zeros((10, 10))
nl = label_noise['70_sym']
for i in range(len(gt_labels)):
  noisy_map[int(nl[i])][int(gt_labels[i])] += 1

#calculate de percentage. Divides each element by the sum of the row
percent_map = noisy_map/noisy_map.sum(axis=0, keepdims=True)
df_cm = pd.DataFrame(percent_map, index = [0,1,2,3,4,5,6,7,8,9],
                  columns = list(range(10)))
plt.figure(figsize = (8,6))

sn.heatmap(df_cm, annot=True)
plt.ylabel('noisy labels')
plt.xlabel('true labels')

unique_elements, counts_elements = np.unique(nl, return_counts=True)
# unique_elements, counts_elements = np.unique(gt_labels, return_counts=True)
plt.bar(unique_elements,counts_elements)<jupyter_output><empty_output><jupyter_text>Train noisy baseline<jupyter_code>#train with 50% noise rate
train_loader.dataset.targets = label_noise['50_sym'].clone()  #update current labels
logs_nb_50_sym = train_test_model(ce_loss)
#train with 50% noise rate
train_loader.dataset.targets = label_noise['70_sym'].clone()  #update current labels
logs_nb_70_sym = train_test_model(ce_loss)
#train with 40% asym. noise rate
train_loader.dataset.targets = label_noise['40_asym'].clone()  #update current labels
logs_nb_40_asym = train_test_model(ce_loss)
fig = plt.figure(figsize=(14,5))
plt.subplot(1,2,1)
plt.plot(logs_baseline['train_acc'], label='train_clean', linewidth=2, linestyle='--',color='k')
plt.plot(logs_nb_50_sym['train_acc'], label='train_50_sym', linewidth=2, linestyle='--', color='r')
plt.plot(logs_nb_70_sym['train_acc'], label='train_70_sym', linewidth=2, linestyle='--', color='g')
plt.plot(logs_nb_40_asym['train_acc'], label='train_40_asym', linewidth=2, linestyle='--', color='b')
plt.plot(logs_baseline['test_acc'], label='test_clean', linewidth=2, color='k')
plt.plot(logs_nb_50_sym['test_acc'], label='test_50_sym', linewidth=2, color='r')
plt.plot(logs_nb_70_sym['test_acc'], label='test_70_sym', linewidth=2, color='g')
plt.plot(logs_nb_40_asym['test_acc'], label='test_40_asym', linewidth=2, color='b')
# plt.legend(frameon=False)
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.ylabel("Accuracy", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2),
          fancybox=True, shadow=True, ncol=4)

plt.subplot(1,2,2)
plt.plot(logs_baseline['train_loss'], label='train_clean', linewidth=2, linestyle='--',color='k')
plt.plot(logs_nb_50_sym['train_loss'], label='train_50_sym', linewidth=2, linestyle='--', color='r')
plt.plot(logs_nb_70_sym['train_loss'], label='train_70_sym', linewidth=2, linestyle='--', color='g')
plt.plot(logs_nb_40_asym['train_loss'], label='train_40_asym', linewidth=2, linestyle='--', color='b')
plt.plot(logs_baseline['test_loss'], label='test_clean', linewidth=2, color='k')
plt.plot(logs_nb_50_sym['test_loss'], label='test_50_sym', linewidth=2, color='r')
plt.plot(logs_nb_70_sym['test_loss'], label='test_70_sym', linewidth=2, color='g')
plt.plot(logs_nb_40_asym['test_loss'], label='test_40_asym', linewidth=2, color='b')
# plt.legend(frameon=False)
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.ylabel("Loss", labelpad=15, fontsize=12, color="#333533", fontweight='bold');<jupyter_output><empty_output><jupyter_text>Combating Noisy Labels  Robust Loss<jupyter_code>#Active Passive Loss (ICML 2020)
#paper code: https://github.com/HanxunH/Active-Passive-Losses
class NormalizedCrossEntropy(torch.nn.Module):
    def __init__(self, num_classes=10, scale=1.0):
        super(NormalizedCrossEntropy, self).__init__()
        self.device = device
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.log_softmax(pred, dim=1)
        label_one_hot = torch.nn.functional.one_hot(labels, self.num_classes).float()
        nce = -1 * torch.sum(label_one_hot * pred, dim=1) / (- pred.sum(dim=1))
        return self.scale * nce.mean()
class ReverseCrossEntropy(torch.nn.Module):
    def __init__(self, num_classes, scale=1.0):
        super(ReverseCrossEntropy, self).__init__()
        self.device = device
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        pred = torch.clamp(pred, min=1e-7, max=1.0)
        label_one_hot = torch.nn.functional.one_hot(labels, self.num_classes).float().to(self.device)
        label_one_hot = torch.clamp(label_one_hot, min=1e-4, max=1.0)
        rce = (-1*torch.sum(pred * torch.log(label_one_hot), dim=1))
        return self.scale * rce.mean()
class NCEandRCE(torch.nn.Module):
    def __init__(self, alpha=0.1, beta=1, num_classes=10):
        super(NCEandRCE, self).__init__()
        self.num_classes = num_classes
        self.nce = NormalizedCrossEntropy(scale=alpha, num_classes=num_classes)
        self.rce = ReverseCrossEntropy(scale=beta, num_classes=num_classes)

    def forward(self, pred, labels):
        return self.nce(pred, labels) + self.rce(pred, labels)
class NormalizedFocalLoss(torch.nn.Module):
    def __init__(self, scale=1.0, gamma=0, num_classes=10, alpha=None, size_average=True):
        super(NormalizedFocalLoss, self).__init__()
        self.gamma = gamma
        self.size_average = size_average
        self.num_classes = num_classes
        self.scale = scale

    def forward(self, input, target):
        target = target.view(-1, 1)
        logpt = F.log_softmax(input, dim=1)
        normalizor = torch.sum(-1 * (1 - logpt.data.exp()) ** self.gamma * logpt, dim=1)
        logpt = logpt.gather(1, target)
        logpt = logpt.view(-1)
        pt = torch.autograd.Variable(logpt.data.exp())
        loss = -1 * (1-pt)**self.gamma * logpt
        loss = self.scale * loss / normalizor

        if self.size_average:
            return loss.mean()
        else:
            return loss.sum()
class NFLandRCE(torch.nn.Module):
    def __init__(self, alpha, beta, num_classes=10, gamma=0.5):
        super(NFLandRCE, self).__init__()
        self.num_classes = num_classes
        self.nfl = NormalizedFocalLoss(scale=alpha, gamma=gamma, num_classes=num_classes)
        self.rce = ReverseCrossEntropy(scale=beta, num_classes=num_classes)

    def forward(self, pred, labels):
        return self.nfl(pred, labels) + self.rce(pred, labels)
class MeanAbsoluteError(torch.nn.Module):
    def __init__(self, num_classes, scale=1.0):
        super(MeanAbsoluteError, self).__init__()
        self.device = device
        self.num_classes = num_classes
        self.scale = scale
        return

    def forward(self, pred, labels):
        pred = F.softmax(pred, dim=1)
        label_one_hot = torch.nn.functional.one_hot(labels, self.num_classes).float().to(self.device)
        mae = 1. - torch.sum(label_one_hot * pred, dim=1)
        # Note: Reduced MAE
        # Original: torch.abs(pred - label_one_hot).sum(dim=1)
        # $MAE = \sum_{k=1}^{K} |\bm{p}(k|\bm{x}) - \bm{q}(k|\bm{x})|$
        # $MAE = \sum_{k=1}^{K}\bm{p}(k|\bm{x}) - p(y|\bm{x}) + (1 - p(y|\bm{x}))$
        # $MAE = 2 - 2p(y|\bm{x})$
        #
        return self.scale * mae.mean()
class NCEandMAE(torch.nn.Module):
    def __init__(self, alpha, beta, num_classes=10):
        super(NCEandMAE, self).__init__()
        self.num_classes = num_classes
        self.nce = NormalizedCrossEntropy(scale=alpha, num_classes=num_classes)
        self.mae = MeanAbsoluteError(scale=beta, num_classes=num_classes)

    def forward(self, pred, labels):
        return self.nce(pred, labels) + self.mae(pred, labels)<jupyter_output><empty_output><jupyter_text>Train with APL<jupyter_code>#train with 70% noise rate
train_loader.dataset.targets = label_noise['70_sym'].clone()  #update current labels
criterion =NCEandMAE(alpha=1, beta=1)
logs_apl_70_sym = train_test_model(criterion)
fig = plt.figure(figsize=(14,5))
plt.subplot(1,2,1)
plt.plot(logs_baseline['test_acc'], label='test_baseline_clean', linewidth=2, color='k')
plt.plot(logs_nb_70_sym['test_acc'], label='test_baseline_70_sym', linewidth=2, color='r')
plt.plot(logs_apl_70_sym['test_acc'], label='test__APL_70_sym', linewidth=2, color='b')
# plt.plot(logs_nb_40_asym['test_acc'], label='test_40_asym', linewidth=2, color='b')
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.ylabel("Accuracy", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2),
          fancybox=True, shadow=True, ncol=4)
plt.ylim(70, 100,10)<jupyter_output><empty_output><jupyter_text>small-loss trick<jupyter_code># noisy_labels = label_noise['70_sym']
nl = label_noise['70_sym']
inds_noisy = np.asarray([ind for ind in range(len(nl)) if nl[ind] != gt_labels[ind]])
inds_clean = np.delete(np.arange(len(nl)), inds_noisy)
plt.hist(logs_nb_70_sym['eval_loss'][-1][inds_noisy],label='noisy',bins=100,alpha=0.5)
plt.hist(logs_nb_70_sym['eval_loss'][-1][inds_clean],label='clean',bins=100,alpha=0.5)
plt.xlabel("Normalized loss", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.legend()
avg_loss_clean = [logs_nb_70_sym['eval_loss'][i][inds_clean].mean() for i in range(num_epochs)]
avg_loss_noisy = [logs_nb_70_sym['eval_loss'][i][inds_noisy].mean() for i in range(num_epochs)]
plt.plot(avg_loss_clean, label='clean',linewidth=2)
plt.plot(avg_loss_noisy, label='noisy',linewidth=2)
plt.ylabel("Avg. Loss", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.legend()<jupyter_output><empty_output><jupyter_text>Remove Noisy Samples<jupyter_code>def train_test_filtering(loss_function,trainloader):
  torch.manual_seed(seed)
  net = Net()
  net = net.to(device)
  optimizer = torch.optim.Adam( net.parameters(), lr=lr) 

  logs={'train_loss':[],'train_acc':[], 'test_loss':[], 'test_acc':[],'eval_loss':[]}

  for epoch in range(num_epochs):
    train_loss, train_acc = train_epoch(net, loss_function, trainloader, optimizer)
    
    logs['train_loss'].append(train_loss)
    logs['train_acc'].append(train_acc)
    eval_loss, prob = eval(net, eval_loader)
    logs['eval_loss'].append(eval_loss)
    
    acc_test, loss_test=test(net, loss_function, test_loader)
    logs['test_loss'].append(loss_test)
    logs['test_acc'].append(acc_test)

    if epoch>10:
      pred_idx_clean = (prob > 0.5).nonzero()[0]
      
      train_clean = Subset(train_data, pred_idx_clean)    #get the subset of samples predicted as clean
      clean_loader = torch.utils.data.DataLoader(train_clean,
                                              batch_size = batch_size,
                                              shuffle = True,num_workers=4,pin_memory=True)
      trainloader=clean_loader

    print('Epoch [{}/{}],\ttrain loss:{:.4f},\ttrain acc:{:.2f},\ttest loss:{:.4f},\ttest accuracy:{:.2f}'.format(epoch,num_epochs,train_loss,train_acc, loss_test, acc_test))
  
  return logs
#train with 70% sym noise
train_loader.dataset.targets = label_noise['70_sym'].clone()  #update current labels
logs_remove_70_sym = train_test_filtering(ce_loss,train_loader)
fig = plt.figure(figsize=(14,5))
plt.subplot(1,2,1)
plt.plot(logs_baseline['test_acc'], label='test_baseline_clean', linewidth=2, color='k')
plt.plot(logs_nb_70_sym['test_acc'], label='test_baseline_70_sym', linewidth=2, color='r')
plt.plot(logs_apl_70_sym['test_acc'], label='test__APL_70_sym', linewidth=2, color='b')
plt.plot(logs_remove_70_sym['test_acc'], label='test_remove_70_sym', linewidth=2, color='g')
plt.grid(True, color="#93a1a1", alpha=0.3)
plt.ylabel("Accuracy", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.xlabel("Epoch", labelpad=15, fontsize=12, color="#333533", fontweight='bold');
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2),
          fancybox=True, shadow=True, ncol=4)
plt.ylim(70, 100,10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15246927892418272, 'fraction_numerical': 0.03088337584048226, 'mean_word_length': 4.4036582310197945, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '617200', 'license': 'MIT', 'n_tokens_mistral': 8093, 'n_tokens_neox': 7618, 'n_words': 1347, 'path': 'codes/tutorial_sibgrapi20.ipynb', 'repo_name': 'filipe-research/tutorial_noisylabels', 'size': '21584'}