<jupyter_start><jupyter_text>Create data iterators<jupyter_code>batch_size = 64
train_folder, val_folder = get_image_folders()

train_iterator = DataLoader(
    train_folder, batch_size=batch_size, num_workers=4,
    shuffle=True, pin_memory=True
)

val_iterator = DataLoader(
    val_folder, batch_size=256, num_workers=4,
    shuffle=False, pin_memory=True
)

# number of training samples
train_size = len(train_folder.imgs)
train_size
# number of validation samples
val_size = len(val_folder.imgs)
val_size<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from densenet import DenseNet
model = DenseNet()
# load the model from step4
model.load_state_dict(torch.load('model_step4.pytorch_state'))
# create different parameter groups
weights = [
    p for n, p in model.named_parameters()
    if 'conv' in n or 'classifier.weight' in n
]
biases = [model.classifier.bias]
bn_weights = [
    p for n, p in model.named_parameters()
    if 'norm' in n and 'weight' in n
]
bn_biases = [
    p for n, p in model.named_parameters()
    if 'norm' in n and 'bias' in n
]
params = [
    {'params': weights, 'weight_decay': 1e-4},
    {'params': biases},
    {'params': bn_weights},
    {'params': bn_biases}
]
optimizer = optim.SGD(params, lr=5e-5, momentum=0.9, nesterov=True)
loss = nn.CrossEntropyLoss().cuda()
model = model.cuda()<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>n_epochs = 3
n_batches = ceil(train_size/batch_size)

# total number of batches in the train set
n_batches
%%time
def optimization_step_fn(model, loss, x_batch, y_batch):
    return optimization_step(model, loss, x_batch, y_batch, optimizer)

all_losses = train(
    model, loss, optimization_step_fn,
    train_iterator, val_iterator, n_epochs
)
# epoch logloss  accuracy    top5_accuracy time  (first value: train, second value: val)<jupyter_output>0  1.257 1.029  0.676 0.734  0.882 0.910  2051.787
1  1.245 1.017  0.679 0.736  0.884 0.911  2033.188
2  1.226 1.014  0.683 0.736  0.887 0.913  2035.694
CPU times: user 1h 32min 32s, sys: 9min 22s, total: 1h 41min 54s
Wall time: 1h 42min<jupyter_text>Save<jupyter_code>model.cpu();
torch.save(model.state_dict(), 'model_step5.pytorch_state')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11291048133153396, 'fraction_numerical': 0.05847953216374269, 'mean_word_length': 4.994609164420485, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '119176', 'license': 'MIT', 'n_tokens_mistral': 927, 'n_tokens_neox': 855, 'n_words': 202, 'path': 'vanilla_densenet_big/train_step5.ipynb', 'repo_name': 'dan-fritchman/trained-ternary-quantization', 'size': '2223'}