<jupyter_start><jupyter_text>Epoch<jupyter_code>model = make_model()
model.to(trainer.device)<jupyter_output><empty_output><jupyter_text>Lambda = 1e-10<jupyter_code>optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch")
logger = Logger("SGD_momentum2.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9908958333333333, 0.03196341497482111)
Valid: (0.9836666666666667, 0.058639409329742195)<jupyter_text>Lambda = 1e-5<jupyter_code>optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch", lambda_=1e-5)
logger = Logger("SGD_momentum_lambda=1e-5_2.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9976875, 0.010387330086668953)
Valid: (0.9855, 0.05503412218927406)<jupyter_text>Lambda = 1e-2<jupyter_code>optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch", lambda_=1e-2)
logger = Logger("SGD_momentum_lambda=1e-2.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9904583333333333, 0.032220454464045666)
Valid: (0.9824166666666667, 0.05974859910442804)<jupyter_text>Epoch average<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg")
logger = Logger("SGD_momentum-avg.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9853333333333333, 0.050679075373957556)
Valid: (0.98, 0.06859579290946324)<jupyter_text>Epoch average, with span = 100<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha=(2 / (100 + 1)))
logger = Logger("SGD_momentum-avg_span_100.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9909166666666667, 0.030257200048925976)
Valid: (0.9831666666666666, 0.06084559714452674)<jupyter_text>Epoch average, with span = 50<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha=(2 / (50 + 1)))
logger = Logger("SGD_momentum-avg_span_50.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9910416666666667, 0.03172329428357382)
Valid: (0.9831666666666666, 0.059001391724838564)<jupyter_text>Epoch average, with span = 20<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha=(2 / (20 + 1)))
logger = Logger("SGD_momentum-avg_span_20.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9903541666666666, 0.033172125154950965)
Valid: (0.98325, 0.05807564333003635)<jupyter_text>Epoch average, with span = 15<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha=(2 / (15 + 1)))
logger = Logger("SGD_momentum-avg_span_15.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.990375, 0.03216092803166248)
Valid: (0.98375, 0.056568290886934845)<jupyter_text>Epoch average, with span = 10<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha=(2 / (10 + 1)))
logger = Logger("SGD_momentum-avg_span_10.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9892083333333334, 0.03670764370728284)
Valid: (0.982, 0.06022620059705029)<jupyter_text>Epoch average, with span = 5<jupyter_code>model = make_model()
model.to(trainer.device)
optimizer = AcceleratedSGD(model.parameters(), 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch_avg", avg_alpha = (2 / (5 + 1)))
logger = Logger("SGD_momentum-avg_span_5.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.to(trainer.device)
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9910208333333334, 0.03151546738048395)
Valid: (0.9838333333333333, 0.056970573332936814)<jupyter_text>Split + epoch<jupyter_code>model = make_model()
model.to(trainer.device)
None
groups = [{"params": [param]} for param in model.parameters()]
optimizer = AcceleratedSGD(groups, 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch")
logger = Logger("SGD_momentum-split.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.98975, 0.03377577416237909)
Valid: (0.9814166666666667, 0.06653467167946898)<jupyter_text>Linear only<jupyter_code>model = make_model()
model.to(trainer.device)
None
conv_group = {
    "params": [param for child in list(model.children())[:10] for param in child.parameters()],
    "method": None
}
fc_group = {
    "params": [param for child in list(model.children())[10:] for param in child.parameters()]
}
groups = [conv_group, fc_group]
optimizer = AcceleratedSGD(groups, 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch")
logger = Logger("SGD_momentum-linear_only.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9907291666666667, 0.03270646293833852)
Valid: (0.9835, 0.05615483507156993)<jupyter_text>Linear + conv separately<jupyter_code>model = make_model()
model.to(trainer.device)
None
conv_group = {
    "params": [param for child in list(model.children())[:10] for param in child.parameters()],
}
fc_group = {
    "params": [param for child in list(model.children())[10:] for param in child.parameters()]
}
groups = [conv_group, fc_group]
optimizer = AcceleratedSGD(groups, 1e-3, k=10, momentum=0.5, weight_decay=1e-5, mode="epoch")
logger = Logger("SGD_momentum-linear_conv.txt")
epochs = 30

for epoch in range(epochs):
    train_loss = trainer.train_epoch(model, optimizer, dl["train"])
    optimizer.finish_epoch()
    val_acc, val_loss = trainer.validation(model, dl["valid"])
    logger.log("Epoch", epoch+1, "|", 
          f"Training loss: {train_loss:.4f}, validation accuracy: {val_acc:.4f}, validation loss: {val_loss:.4f}")
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)
optimizer.accelerate()
optimizer.store_parameters()
model.cuda()
None
train_score = trainer.validation(model, dl["train"])
valid_score = trainer.validation(model, dl["valid"])
logger.log("Train:", train_score)
logger.log("Valid:", valid_score)<jupyter_output>Train: (0.9900208333333333, 0.03382842983398587)
Valid: (0.98275, 0.06026328789287557)
{'fraction_non_alphanumeric': 0.1652162618521886, 'fraction_numerical': 0.06884010910507858, 'mean_word_length': 6.962254395036195, 'pattern_counts': {'":': 6, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 16, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '298031', 'license': 'MIT', 'n_tokens_mistral': 6120, 'n_tokens_neox': 5478, 'n_words': 967, 'path': 'notebooks/nn/SGD_momentum.ipynb', 'repo_name': 'JaworWr/MLAcceleration', 'size': '15398'}