<jupyter_start><jupyter_text>MNIST example with 3-conv. layer networkThis example demonstrates the usage of `FastaiLRFinder` with a 3-conv. layer network on the MNIST dataset.<jupyter_code>%matplotlib inline

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torchvision.datasets import MNIST<jupyter_output><empty_output><jupyter_text>Loading MNIST<jupyter_code>mnist_pwd = "<PASSWORD>"
batch_size= 256
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])

trainset = MNIST(mnist_pwd, train=True, download=True, transform=transform)
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4)

testset = MNIST(mnist_pwd, train=False, download=True, transform=transform)
testloader = DataLoader(testset, batch_size=batch_size * 2, shuffle=False, num_workers=0)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)
from ignite.engine import create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import Loss, Accuracy
from ignite.contrib.handlers import FastaiLRFinder, ProgressBar<jupyter_output><empty_output><jupyter_text>Training loss (fastai)This learning rate test range follows the same procedure used by fastai. The model is trained for `num_iter` iterations while the learning rate is increased from its initial value specified by the optimizer algorithm to `end_lr`. The increase can be linear (`step_mode="linear"`) or exponential (`step_mode="exp"`); linear provides good results for small ranges while exponential is recommended for larger ranges.<jupyter_code>device = "cuda" if torch.cuda.is_available() else "cpu"
criterion = nn.NLLLoss()
model = Net()
model.to(device)  # Move model before creating optimizer
optimizer = optim.SGD(model.parameters(), lr=3e-4, momentum=0.9)
trainer = create_supervised_trainer(model, optimizer, criterion, device=device)
ProgressBar(persist=True).attach(trainer, output_transform=lambda x: {"batch loss": x})

lr_finder = FastaiLRFinder()
to_save={'model': model, 'optimizer': optimizer}
with lr_finder.attach(trainer, to_save, diverge_th=1.5) as trainer_with_lr_finder:
    trainer_with_lr_finder.run(trainloader)
    
trainer.run(trainloader, max_epochs=10)

evaluator = create_supervised_evaluator(model, metrics={"acc": Accuracy(), "loss": Loss(nn.NLLLoss())}, device=device)
evaluator.run(testloader)

print(evaluator.state.metrics)
lr_finder.plot()
lr_finder.lr_suggestion()<jupyter_output><empty_output><jupyter_text>Let's now setup suggested learning rate to the optimizer and we can train the model with optimal learning rate.*Note, that model and optimizer were restored to their initial states when `FastaiLRFinder` finished.*<jupyter_code>optimizer.param_groups[0]['lr'] = lr_finder.lr_suggestion()

trainer.run(trainloader, max_epochs=10)
evaluator.run(testloader)
print(evaluator.state.metrics)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11187277214148615, 'fraction_numerical': 0.01891965999451604, 'mean_word_length': 5.975143403441683, 'pattern_counts': {'":': 3, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '45396', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1209, 'n_words': 287, 'path': 'examples/notebooks/FastaiLRFinder_MNIST.ipynb', 'repo_name': 'willfrey/ignite', 'size': '3641'}