<jupyter_start><jupyter_text>Data<jupyter_code>import collections
import torch
import torchvision
import torchvision.transforms as transforms


bs = 32
num_workers = 4

data_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])


loaders = collections.OrderedDict()

trainset = torchvision.datasets.CIFAR10(
    root='./data', train=True,
    download=True, transform=data_transform)
trainloader = torch.utils.data.DataLoader(
    trainset, batch_size=bs,
    shuffle=True, num_workers=num_workers)

testset = torchvision.datasets.CIFAR10(
    root='./data', train=False,
    download=True, transform=data_transform)
testloader = torch.utils.data.DataLoader(
    testset, batch_size=bs,
    shuffle=False, num_workers=num_workers)

loaders["train"] = trainloader
loaders["valid"] = testloader<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x<jupyter_output><empty_output><jupyter_text>Intro@TODO<jupyter_code># for graphs use `tensorboard --logdir=./logs`
from catalyst.dl.utils import UtilsFactory<jupyter_output><empty_output><jupyter_text>Setup 1 - typical training<jupyter_code>from catalyst.dl.experiments import SupervisedRunner

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_1"

# model, criterion, optimizer
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    loaders=loaders,
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)
# you can use plotly and tensorboard to plot metrics inside jupyter
# by default it only plots loss
UtilsFactory.plot_metrics(logdir=logdir)<jupyter_output><empty_output><jupyter_text>Setup 2 - training with sheduler<jupyter_code>from catalyst.dl.experiments import SupervisedRunner

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_2"

# model, criterion, optimizer
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())

# any Pytorch scheduler supported
# scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3, 8], gamma=0.3)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.5, patience=2)

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    scheduler=scheduler,
    loaders=loaders,
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)<jupyter_output><empty_output><jupyter_text>Setup 3 - training with early stop<jupyter_code>from catalyst.dl.experiments import SupervisedRunner
from catalyst.dl.callbacks import EarlyStoppingCallback

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_3"

# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3, 8], gamma=0.3)

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    scheduler=scheduler,
    loaders=loaders,
    callbacks=[
        EarlyStoppingCallback(patience=2, min_delta=0.01)
    ],
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)
UtilsFactory.plot_metrics(logdir=logdir, metrics=["loss", "base/lr"])<jupyter_output><empty_output><jupyter_text>Setup 4 - training with additional metrics<jupyter_code>from catalyst.dl.experiments import SupervisedRunner
from catalyst.dl.callbacks import EarlyStoppingCallback, PrecisionCallback

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_4"

# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3, 8], gamma=0.3)

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    scheduler=scheduler,
    loaders=loaders,
    callbacks=[
        PrecisionCallback(),
        EarlyStoppingCallback(patience=2, min_delta=0.01)
    ],
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)
UtilsFactory.plot_metrics(
    logdir=logdir, 
    metrics=["loss", "precision01", "precision03", "base/lr"])<jupyter_output><empty_output><jupyter_text>Setup 5 - training with 1cycle<jupyter_code>from catalyst.dl.experiments import SupervisedRunner
from catalyst.dl.callbacks import EarlyStoppingCallback, PrecisionCallback, OneCycleLR

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_5"

# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    loaders=loaders,
    callbacks=[
        OneCycleLR(
            cycle_len=num_epochs, 
            div_factor=3,
            increase_fraction=0.3,
            momentum_range=(0.95, 0.85)),
        PrecisionCallback(),
        EarlyStoppingCallback(patience=2, min_delta=0.01)
    ],
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)
UtilsFactory.plot_metrics(
    logdir=logdir,
    step="batch",
    metrics=["loss", "precision01", "base/lr", "base/momentum"])<jupyter_output><empty_output><jupyter_text>Setup 6 - pipeline check<jupyter_code>from catalyst.dl.experiments import SupervisedRunner

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_6"

# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3, 8], gamma=0.3)

# model runner
runner = SupervisedRunner()

# model training
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    scheduler=scheduler,
    loaders=loaders,
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True,
    check=True  # here is the trick
)<jupyter_output><empty_output><jupyter_text>Setup 7 - multi-stage training<jupyter_code>from catalyst.dl.experiments import SupervisedRunner
from catalyst.dl.callbacks import EarlyStoppingCallback, PrecisionCallback

# experiment setup
num_epochs = 10
logdir = "./logs/cifar_simple_notebook_7"

# model, criterion, optimizer, scheduler
model = Net()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters())
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3, 8], gamma=0.3)

# model runner
runner = SupervisedRunner()

# model training - 1
runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    scheduler=scheduler,
    loaders=loaders,
    callbacks=[
        PrecisionCallback(),
        EarlyStoppingCallback(patience=2, min_delta=0.01)
    ],
    logdir=logdir,
    num_epochs=num_epochs,
    verbose=True
)

# model training - 2
num_epochs = 5
logdir = "./logs/cifar_simple_notebook_8"
optimizer = torch.optim.SGD(model.parameters(), lr=0.05)

runner.train(
    model=model,
    criterion=criterion,
    optimizer=optimizer,
    loaders=loaders,
    logdir=logdir,
    num_epochs=num_epochs
)<jupyter_output><empty_output><jupyter_text>Setup 8 - loader inference<jupyter_code>from catalyst.dl.callbacks import InferCallback
loaders = collections.OrderedDict([("infer", loaders["train"])])
runner.infer(
    model=model,
    loaders=loaders,
    callbacks=[InferCallback()],
)
runner.callbacks[0].predictions["logits"].shape<jupyter_output><empty_output><jupyter_text>Setup 9 - batch inference<jupyter_code>features, targets = next(iter(loaders["infer"]))
features.shape
runner_in = {runner.input_key: features}
runner_out = runner.predict_batch(runner_in)
runner_out["logits"].shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11521367521367522, 'fraction_numerical': 0.017663817663817662, 'mean_word_length': 4.815772034459907, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '138441', 'license': 'MIT', 'n_tokens_mistral': 3147, 'n_tokens_neox': 2992, 'n_words': 540, 'path': 'examples/notebook-example.ipynb', 'repo_name': 'ferrine/catalyst', 'size': '8775'}