<jupyter_start><jupyter_text>using torch.optim.lr_scheduler to change lr rate in training progress<jupyter_code>##############################################################################
# Define Scheduler
##############################################################################
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, factor=0.1, patience=5, verbose=True
)<jupyter_output><empty_output><jupyter_text>scheduler.step(mean_loss) Reduce learning rate when a metric has stopped improving. for more detail: https://pytorch.org/docs/stable/optim.html?highlight=reducelronplateautorch.optim.lr_scheduler.ReduceLROnPlateau<jupyter_code># Train Network
for epoch in range(1, num_epochs):
    losses = []

    for batch_idx, (data, targets) in enumerate(train_loader):
        # Get data to cuda if possible
        data = data.reshape(data.shape[0], -1)
        data = data.to(device=device)
        targets = targets.to(device=device)

        # forward
        scores = model(data)
        loss = criterion(scores, targets)

        losses.append(loss.item())

        # backward
        loss.backward()

        # gradient descent or adam step
        # scheduler.step(loss)
        optimizer.step()
        optimizer.zero_grad()

    mean_loss = sum(losses) / len(losses)
##############################################################################
    # After each epoch do scheduler.step, note in this scheduler we need to send
    # in loss for that epoch!
    scheduler.step(mean_loss)
##############################################################################
    print(f"Cost at epoch {epoch} is {mean_loss}")<jupyter_output>Cost at epoch 1 is 0.8224168917390583
{'fraction_non_alphanumeric': 0.2625730994152047, 'fraction_numerical': 0.014035087719298246, 'mean_word_length': 4.483974358974359, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '273635', 'license': 'Apache-2.0', 'n_tokens_mistral': 501, 'n_tokens_neox': 469, 'n_words': 108, 'path': '9_Learning_rate_scheduler.ipynb', 'repo_name': 'JiaheXu/Pytorch_tutorial', 'size': '1710'}