<jupyter_start><jupyter_text>Metric function: quadratic weighted kappa<jupyter_code>def quadratic_weighted_kappa(raters, min_rating=0., max_rating=4.):
    """
    Calculates the quadratic weighted kappa
    quadratic_weighted_kappa calculates the quadratic weighted kappa
    value, which is a measure of inter-rater agreement between two raters
    that provide discrete numeric ratings.  Potential values range from -1
    (representing complete disagreement) to 1 (representing complete
    agreement).  A kappa value of 0 is expected if all agreement is due to
    chance.
    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b
    each correspond to a list of integer ratings.  These lists must have the
    same length.
    The ratings should be integers, and it is assumed that they contain
    the complete range of possible ratings.
    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating
    is the minimum possible rating, and max_rating is the maximum possible
    rating
    """
    rater_a, rater_b = raters
    rater_a = np.array(rater_a, dtype=int)
    rater_b = np.array(rater_b, dtype=int)
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
        min_rating = min(min(rater_a), min(rater_b))
    if max_rating is None:
        max_rating = max(max(rater_a), max(rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
                                min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
        for j in range(num_ratings):
            expected_count = (hist_rater_a[i] * hist_rater_b[j]
                              / num_scored_items)
            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)
            numerator += d * conf_mat[i][j] / num_scored_items
            denominator += d * expected_count / num_scored_items

    return 1.0 - numerator / denominator<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>df_train = pd.read_csv('/storage/aptosplus/trainLabels_smallds.csv')
df_valid = pd.read_csv('/storage/aptosplus/validLabels_smallds.csv')

train_x = df_train[['id_code', 'ds']]
train_y = df_train['diagnosis']

valid_x = df_valid[['id_code', 'ds']]
valid_y = df_valid['diagnosis']
train_y.hist()
valid_y.hist()
valid_labels_count = Counter()

for i in valid_y:
    valid_labels_count.update([i])

valid_labels_count
train_labels_count = Counter()

for i in train_y:
    train_labels_count.update([i])

train_labels_count
batch_size=16
model_name = 'efficientnet-b1'
num_classes = 1
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# pytorch training dataset & loader
ds = ImageDS(train_x, train_y, small=True)
loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=8)

# pytorch cross-validation dataset & loader
ds_val = ImageDS(valid_x, valid_y, small=True)
val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=8)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>model = EfficientNet.from_name(model_name) 
in_features = model._fc.in_features
model._fc = nn.Linear(in_features, num_classes)
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=10e-4, momentum=0.9)

metrics = {
    'loss': Loss(criterion),
    'accuracy_qwk': Accuracy(quadratic_weighted_kappa)
}

if torch.cuda.device_count() > 1:
  print("Let's use", torch.cuda.device_count(), "GPUs!")
  model = nn.DataParallel(model)

trainer = create_supervised_trainer(model, optimizer, criterion, device=device)
val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)
def adjust_learning_rate(optimizer, epoch):
    lr = 10e-4

    if epoch < 80:
        lr = 10e-4
    elif epoch < 150:
        lr = 10e-5
    elif epoch < 190:
        lr = 50e-6
    else:
        lr = 10e-6

    print(f'\nLearning rate of epoch {epoch + 1}: {lr}\n')
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

@trainer.on(Events.EPOCH_COMPLETED)
def update_lr_scheduler(engine):
    adjust_learning_rate(optimizer, engine.state.epoch)
    lr = float(optimizer.param_groups[0]['lr'])
def get_saved_model_path(epoch):
    return f'/storage/Model_{model_name}_{epoch}.pth'

best_loss = 100.
best_epoch = 1
best_epoch_file = ''

@trainer.on(Events.EPOCH_COMPLETED)
def save_best_epoch_only(engine):
    epoch = engine.state.epoch

    global best_loss
    global best_epoch
    global best_epoch_file
    best_loss = 100. if epoch == 1 else best_loss
    best_epoch = 1 if epoch == 1 else best_epoch
    best_epoch_file = '' if epoch == 1 else best_epoch_file

    metrics = val_evaluator.run(val_loader).metrics

    print(f'\nLoss for epoch {epoch}: {metrics["loss"]}\n')
    print(f'\nAccuracy for epoch {epoch}: {metrics["accuracy_qwk"]}\n')
    if metrics['loss'] < best_loss:
        prev_best_epoch_file = get_saved_model_path(best_epoch)
        if os.path.exists(prev_best_epoch_file):
            os.remove(prev_best_epoch_file)
            
        best_loss = metrics['loss']
        best_epoch = epoch
        best_epoch_file = get_saved_model_path(best_epoch)
        print(f'\nEpoch: {best_epoch} - New best loss! Loss: {best_loss}\n\n\n')
        torch.save(model.state_dict(), best_epoch_file)
# scheduler = CosineAnnealingScheduler(optimizer, 'lr', 10e-4, 10e-6, len(loader))
# trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
pbar = ProgressBar(bar_format='')
pbar.attach(trainer, output_transform=lambda x: {'loss': x})
print('Training started')
trainer.run(loader, max_epochs=150)<jupyter_output>Training started
{'fraction_non_alphanumeric': 0.1066827005669129, 'fraction_numerical': 0.01271259233808624, 'mean_word_length': 4.058210251954822, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '177725', 'license': 'MIT', 'n_tokens_mistral': 2107, 'n_tokens_neox': 2036, 'n_words': 437, 'path': 'training.ipynb', 'repo_name': 'MichelML/ml-aptos', 'size': '5821'}