<jupyter_start><jupyter_text>Import modules<jupyter_code>import copy
import matplotlib.pyplot as plt
import os
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
import torchvision
from torchvision import datasets, models, transforms<jupyter_output><empty_output><jupyter_text>Utils<jupyter_code>def display_formatted_time(elapsed_time, msg=""):
    minutes, seconds = map(int, divmod(elapsed_time, 60));
    print("Elapsed time - {0}: {1}min {2}s".format(msg, minutes, seconds))<jupyter_output><empty_output><jupyter_text>Define the network<jupyter_code>class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)  # 
        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
        self.conv2_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(320, 50)
        self.fc2 = nn.Linear(50, 10)
        
    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 320)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

print(Net())
def train(model, device, train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)  # Negative Log Likelihood Loss
        loss.backward()
        optimizer.step()
        if batch_idx % log_interval == 0:
            print("Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()
            ))

def test(model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction="sum").item()
            pred = output.max(1, keepdim=True)[1]
            correct += pred.eq(target.view_as(pred)).sum().item()

        test_loss /= len(test_loader.dataset)
        print("\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format(
            test_loss, correct, len(test_loader.dataset),
            100. * correct / len(test_loader.dataset)
        ))<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
print("Device: ", device)

torch.manual_seed(1)

kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}


batch_size = 64
test_batch_size = 1000
epochs = 10
log_interval = 10
lr = 0.01
momentum = 0.5<jupyter_output>Device:  cuda<jupyter_text>Load data<jupyter_code>train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../../data',
                                train=True,
                                download=True,
                                transform=transforms.Compose([
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))
                                ])
    ),
    batch_size=batch_size,
    shuffle=True,
    **kwargs
)

test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../../data',
                                train=False,
                                transform=transforms.Compose([
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))
                                ])
    ),
    batch_size=test_batch_size,
    shuffle=True,
    **kwargs
)<jupyter_output><empty_output><jupyter_text>Baseline model Train / Test<jupyter_code>since = time.time()

model = Net().to(device)
optimizer = optim.SGD(model.parameters(),
                                         lr=lr,
                                         momentum=momentum
                      )

for epoch in range(1, epochs + 1):
    train(model, device, train_loader, optimizer, epoch)
    test(model, device, test_loader)

display_formatted_time(time.time() - since)<jupyter_output>Train Epoch: 1 [0/60000 (0%)]	Loss: 2.373651
Train Epoch: 1 [640/60000 (1%)]	Loss: 2.310517
Train Epoch: 1 [1280/60000 (2%)]	Loss: 2.281828
Train Epoch: 1 [1920/60000 (3%)]	Loss: 2.315809
Train Epoch: 1 [2560/60000 (4%)]	Loss: 2.235439
Train Epoch: 1 [3200/60000 (5%)]	Loss: 2.234249
Train Epoch: 1 [3840/60000 (6%)]	Loss: 2.226109
Train Epoch: 1 [4480/60000 (7%)]	Loss: 2.228646
Train Epoch: 1 [5120/60000 (9%)]	Loss: 2.132810
Train Epoch: 1 [5760/60000 (10%)]	Loss: 2.113178
Train Epoch: 1 [6400/60000 (11%)]	Loss: 2.030113
Train Epoch: 1 [7040/60000 (12%)]	Loss: 1.877118
Train Epoch: 1 [7680/60000 (13%)]	Loss: 1.894014
Train Epoch: 1 [8320/60000 (14%)]	Loss: 1.725610
Train Epoch: 1 [8960/60000 (15%)]	Loss: 1.739437
Train Epoch: 1 [9600/60000 (16%)]	Loss: 1.533461
Train Epoch: 1 [10240/60000 (17%)]	Loss: 1.549235
Train Epoch: 1 [10880/60000 (18%)]	Loss: 1.498123
Train Epoch: 1 [11520/60000 (19%)]	Loss: 1.502510
Train Epoch: 1 [12160/60000 (20%)]	Loss: 1.326280
Train Epoch: 1 [12800/60000 ([...]<jupyter_text>___  LR Scheduler<jupyter_code>optimizer_ft = optim.SGD(model.parameters(),
                                              lr=lr,
                                              momentum=momentum)
scheduler = lr_scheduler.StepLR(optimizer=optimizer_ft,
                                                         step_size=2.,  # {step} epoch ごとにLRを減少させる
                                                         gamma=0.1)  # step ごとに lr = lr * gamma
since = time.time()

model = Net().to(device)
optimizer = optim.SGD(model.parameters(),
                                         lr=lr,
                                         momentum=momentum)

for epoch in range(1, epochs + 1):
    scheduler.step()
    train(model, device, train_loader, optimizer, epoch)
    test(model, device, test_loader)

display_formatted_time(time.time() - since)<jupyter_output>Train Epoch: 1 [0/60000 (0%)]	Loss: 2.354365
Train Epoch: 1 [640/60000 (1%)]	Loss: 2.297701
Train Epoch: 1 [1280/60000 (2%)]	Loss: 2.291977
Train Epoch: 1 [1920/60000 (3%)]	Loss: 2.260900
Train Epoch: 1 [2560/60000 (4%)]	Loss: 2.262597
Train Epoch: 1 [3200/60000 (5%)]	Loss: 2.283646
Train Epoch: 1 [3840/60000 (6%)]	Loss: 2.213664
Train Epoch: 1 [4480/60000 (7%)]	Loss: 2.182343
Train Epoch: 1 [5120/60000 (9%)]	Loss: 2.219697
Train Epoch: 1 [5760/60000 (10%)]	Loss: 2.101493
Train Epoch: 1 [6400/60000 (11%)]	Loss: 2.057653
Train Epoch: 1 [7040/60000 (12%)]	Loss: 2.111837
Train Epoch: 1 [7680/60000 (13%)]	Loss: 1.852859
Train Epoch: 1 [8320/60000 (14%)]	Loss: 1.800821
Train Epoch: 1 [8960/60000 (15%)]	Loss: 1.659863
Train Epoch: 1 [9600/60000 (16%)]	Loss: 1.386412
Train Epoch: 1 [10240/60000 (17%)]	Loss: 1.407535
Train Epoch: 1 [10880/60000 (18%)]	Loss: 1.320320
Train Epoch: 1 [11520/60000 (19%)]	Loss: 1.579342
Train Epoch: 1 [12160/60000 (20%)]	Loss: 1.304948
Train Epoch: 1 [12800/60000 ([...]<jupyter_text>___  Transfer learning実行せず書き方のみ参照すること<jupyter_code>batch_size = 64
test_batch_size = 10
epochs = 10
log_interval = 10
lr = 0.01
momentum = 0.9
train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../../data',
                                train=True,
                                download=True,
                                transform=transforms.Compose([
                                    transforms.RandomResizedCrop(224),
                                    transforms.RandomHorizontalFlip(),
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))
                                ])
    ),
    batch_size=batch_size,
    shuffle=True,
    **kwargs
)

test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../../data',
                                train=False,
                                transform=transforms.Compose([
                                    transforms.Resize(256),
                                    transforms.CenterCrop(224),
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.1307,), (0.3081,))
                                ])
    ),
    batch_size=test_batch_size,
    shuffle=True,
    **kwargs
)<jupyter_output><empty_output><jupyter_text>Finetuning<jupyter_code>model_ft = models.resnet18(pretrained=True)
num_ftrs = model_ft.fc.in_features

# Input, Output をMNISTに合わせる
model_ft.conv1 = nn.Conv2d(in_channels=1,
                                                    out_channels=64,
                                                    kernel_size=7,
                                                    stride=2,
                                                    padding=3,
                                                    bias=False)
model_ft.fc = nn.Linear(num_ftrs, 10)

model_ft = model_ft.to(device)
print(model_ft)

# criterion = nn.CrossEntropyLoss()

optimizer_ft = optim.SGD(model_ft.parameters(),
                                              lr=lr,
                                              momentum=momentum)

exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
                                                                      step_size=2,
                                                                      gamma=0.1)<jupyter_output>ResNet(
  (conv1): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu): ReLU(inplace)
  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (layer1): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
      (conv2): Co[...]<jupyter_text>Train and evaluate<jupyter_code>since = time.time()

for epoch in range(1, epochs + 1):
    exp_lr_scheduler.step()
    train(model_ft, device, train_loader, optimizer_ft, epoch)
    test(model_ft, device, test_loader)

display_formatted_time(time.time() - since)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13427433948606587, 'fraction_numerical': 0.0922005066956207, 'mean_word_length': 2.140949133276499, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '407759', 'license': 'MIT', 'n_tokens_mistral': 4478, 'n_tokens_neox': 3741, 'n_words': 774, 'path': 'notebooks/PyTorch/pytorch_mnist.ipynb', 'repo_name': 'shitake/ml-sandbox', 'size': '11052'}