<jupyter_start><jupyter_text>Momentum<jupyter_code>optimizer = AcceleratedSGD(model.parameters(), 1e-1, k=10, momentum=0.9, weight_decay=1e-5, lambda_=1e-8)
loss_fn = nn.NLLLoss()
epochs = 20

for epoch in range(epochs):
    print("Epoch", epoch+1)
    loss_log = []
    train_epoch(model, loss_log)
    print(f"Training loss: {np.mean(loss_log):.4f}")
    optimizer.finish_epoch()
    val_acc, val_loss = validation(model, valid_loader)
    print(f"Validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
    print("Epoch", epoch+1, 
          f"Training loss: {np.mean(loss_log):.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}",
          file=log_file, flush=True
         )
train_score = validation(model, train_loader)
valid_score = validation(model, valid_loader)
print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RNA"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RRE"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
final_state = deepcopy(model.state_dict())
# final_optim_state = deepcopy(optimizer.state_dict())

epochs = 10
print("Additional training", file=log_file)

for epoch in range(epochs):
    print("Epoch", epoch+1)
    loss_log = []
    train_epoch(model_acc, loss_log)
    print(f"Training loss: {np.mean(loss_log):.4f}")
    optimizer.finish_epoch()
    val_acc, val_loss = validation(model_acc, valid_loader)
    print(f"Validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
    print("Epoch", epoch+1, 
          f"Training loss: {np.mean(loss_log):.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}",
          file=log_file, flush=True
         )
model.load_state_dict(final_state)
# optimizer.load_state_dict(final_optim_state)
model.cuda()
optimizer.param_groups[0]["method"] = "RNA"
optimizer.param_groups[0]["lr"] = 1e-2
print("Main training", file=log_file)
epochs = 25

for epoch in range(epochs):
    print("Epoch", epoch+1)
    loss_log = []
    train_epoch(loss_log)
    print(f"Training loss: {np.mean(loss_log):.4f}")
    optimizer.finish_epoch()
    val_acc, val_loss = validation(model, valid_loader)
    print(f"Validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
    print("Epoch", epoch+1, 
          f"Training loss: {np.mean(loss_log):.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}",
          file=log_file, flush=True
         )
train_score = validation(model, train_loader)
valid_score = validation(model, valid_loader)
print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RNA"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RRE"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RNA"
optimizer.param_groups[0]["lr"] = 1e-3
epochs = 25

for epoch in range(epochs):
    print("Epoch", epoch+1)
    loss_log = []
    train_epoch(loss_log)
    print(f"Training loss: {np.mean(loss_log):.4f}")
    optimizer.finish_epoch()
    val_acc, val_loss = validation(model, valid_loader)
    print(f"Validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
    print("Epoch", epoch+1, 
          f"Training loss: {np.mean(loss_log):.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}",
          file=log_file, flush=True
         )
train_score = validation(model, train_loader)
valid_score = validation(model, valid_loader)
print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RNA"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)
optimizer.param_groups[0]["method"] = "RRE"
print(optimizer.param_groups[0]["method"], flush=True, file=log_file)
model.cpu()
model_acc = deepcopy(model)
optimizer.accelerate()
optimizer.store_parameters([model_acc.parameters()])
model_acc.cuda()
train_score = validation(model_acc, train_loader)
valid_score = validation(model_acc, valid_loader)
model_acc.cpu()
model.cuda()

print("Train:", train_score)
print("Valid:", valid_score)
print("Train:", train_score, flush=True, file=log_file)
print("Valid:", valid_score, flush=True, file=log_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15935906773488712, 'fraction_numerical': 0.010196649672250545, 'mean_word_length': 7.115839243498818, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '563072', 'license': 'MIT', 'n_tokens_mistral': 2463, 'n_tokens_neox': 2414, 'n_words': 420, 'path': 'notebooks/nn_large/resnet-augmentation-comparison.ipynb', 'repo_name': 'JaworWr/MLAcceleration', 'size': '6865'}