<jupyter_start><jupyter_text>Imports<jupyter_code>import copy
import os
import sys
import time

import matplotlib.pyplot as plt
import numpy as np
from pathlib import Path
import torch
from torch.m
import torchvision
from torchvision import transforms 
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
from torch import nn
from torch import optim
from torch.optim.lr_scheduler import StepLR
import torch.nn.functional as F

sys.path.append('../')
import utils

%matplotlib inline
plt.style.use('gadfly')
%reload_ext autoreload
%autoreload 2
torch.__version__<jupyter_output><empty_output><jupyter_text>Get Data<jupyter_code>PATH = Path('../data/hymenoptera_data/')
BATCH_SZ = 4
EPOCHS = 10
N_CLASSES = 2
LEARNING_RATE = 1e-3
ls {PATH}
means = [0.485, 0.456, 0.406]
stds = [0.229, 0.224, 0.225]

tfms = {
    'train': transforms.Compose([transforms.RandomResizedCrop(224),
                                transforms.RandomHorizontalFlip(),
                                transforms.ToTensor(),
                                transforms.Normalize(means, stds)]),
    'valid': transforms.Compose([transforms.Resize(256),
                                transforms.CenterCrop(224),
                                transforms.ToTensor(),
                                transforms.Normalize(means, stds)])
}
imgs_ds = {s: ImageFolder(PATH / s, tfms[s]) for s in ['train', 'valid']}
imgs_dl = {s: DataLoader(imgs_ds[s], BATCH_SZ, shuffle=True, num_workers=4)
           for s in ['train', 'valid']}
len(imgs_ds['train']), len(imgs_ds['valid'])
imgs_ds['train'].classes, imgs_ds['train'].class_to_idx
# Sample images
images, labels = next(iter(imgs_dl['train']))
titles = [imgs_ds['train'].classes[label] for label in labels]
utils.show_images(torchvision.utils.make_grid(images), means, stds, title=titles)
device = ('cuda:0' if torch.cuda.is_available() else 'cpu')
device<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>def train_model(
    model, criterion, optimizer, data_loaders, dataset_sizes, device,
    lr_scheduler, n_epochs):
    start = time.time()
    
    best_model_weights = copy.deepcopy(model.state_dict())
    best_accuracy = 0
    history = {
        'loss': {'train': [], 'valid': []},
        'accuracy': {'train': [], 'valid': []}
    }

    print(f'{30 * "*":>50} Started training {30 * "*":<50}')
    for epoch in range(n_epochs):
        print(f'Epoch {(epoch + 1)}/{n_epochs}\n{10 * "-"}')
        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0
            running_accuracy = 0

            for images, labels in data_loaders[phase]:
                images = images.to(device)
                labels = labels.to(device)
              
                # Zero grads
                optimizer.zero_grad()
                
                # Fwd pass
                with torch.set_grad_enabled(phase == 'train'):    # Set gradients on and off
                    output = model(images)
                    pred_labels = torch.argmax(output, 1)
                    
                    # Compute loss
                    loss = criterion(output, labels)

                    # Compute gradients if it's training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                running_loss += loss.item()
                running_accuracy += (pred_labels == labels).sum().item()
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_accuracy = running_accuracy / dataset_sizes[phase]
            print(f'{phase.capitalize():<5} Loss : {epoch_loss:.4f}, '
                  f'Accuracy : {epoch_accuracy:.2%}')

            # Deep copy best model
            if phase == 'valid'and epoch_accuracy > best_accuracy:
                best_accuracy = epoch_accuracy
                best_model_weights = copy.deepcopy(model.state_dict())
            
            # Update losses and accuracies
            history['loss'][phase].append(epoch_loss)
            history['accuracy'][phase].append(epoch_accuracy)
            
        print()
    time_elapsed = time.time() - start
    print(f'Training complete in : {time_elapsed // 60}m and {time_elapsed % 60:.0f}s')
    print(f'Best validation accuracy : {best_accuracy:.2%}')
    model.load_state_dict(best_model_weights)
    
    return model, history
model.state_dict().keys()
torch.argmax(torch.tensor([[1, 2], [3, 1]]), dim=1)
torch.topk(torch.tensor([[1, 2], [3, 1]]), 1, dim=1)
class Net(nn.Module):
    
    def __init__(self, input_size, hidden_layers, output_size):
        super().__init__()
        self.linears = nn.ModuleList([nn.Linear(input_size, hidden_layers[0])])
        hidden_layers_sizes = zip(hidden_layers[:-1], hidden_layers[1:])
        self.linears.extend([nn.Linear(h1, h2) for h1, h2 in hidden_layers_sizes])
        self.output = nn.Linear(hidden_layers[-1], output_size)
    
    def forward(self, x):
        for m in self.linears:
            x = F.relu(m(x))
        return F.log_softmax(x, dim=1)
from torch.utils.data import TensorDataset
ds = TensorDataset(torch.tensor([[1, 2], [3, 4]]), torch.tensor([0, 1]))
dl = DataLoader(ds)
next(iter(ds))
model = Net(10, [20, 30, 40], 10)
model
model.linears[0]
from collections import OrderedDict
class Net(nn.Module):
    
    def __init__(self):
        super().__init__()
        self.linears = nn.Sequential(nn.ModuleDict(
            OrderedDict([
                ('fc1', nn.Linear(10, 20)),
                ('fc2', nn.Linear(20, 30))
                ])))
        self.nonlinears = nn.Sequential(nn.ModuleDict(
            OrderedDict([
                ('fc1', nn.Linear(30, 40)),
                ('fc2', nn.Linear(40, 50))
                ])))
        
    def forward(self, x):
        x = self.linears(x)
        x = self.nonlinears(x)
        return F.log_softmax(x, dim=1)
model = Net()
model
model.linears[0]
m.fc1.weight.data
len([param for param in model.parameters() if param.requires_grad == True])
conv_base.layer1[0].conv1.weight.data[0, 0]<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>model = torchvision.models.resnet18(pretrained=True)
model
# Freeze conv layers
for param in model.parameters():
    param.requires_grad = False
# Add new classifier
model.fc = nn.Linear(512, N_CLASSES)
model
print(f'Number of trainable parameters : '
      f'{len([param for param in model.parameters() if param.requires_grad])}')
print(f'Number of non-trainable parameters : '
      f'{len([param for param in model.parameters() if not param.requires_grad])}')
model.layer1[0].conv1.weight.data[0, 0], \
model.layer1[0].conv1.weight.data[0, 1]
criterion = nn.CrossEntropyLoss(reduction='sum')
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)
model.to(device)
best_weights = copy.deepcopy(model.state_dict())
best_acc = 0
history = {
    'loss': {'train': [], 'valid': []},
    'accuracy': {'train': [], 'valid': []}
}

for epoch in range(EPOCHS):
    print(f'Epoch : {epoch + 1}/{EPOCHS}\n{"-" * (len("epoch") + 7)}')
    for phase in ['train', 'valid']:
        running_loss = 0
        running_acc = 0
        
        if phase == 'train':
          model.train()
        else:
          model.eval()
        
        for images, labels in imgs_dl[phase]:
            images = images.to(device)
            labels = labels.to(device)
          
            with torch.set_grad_enabled(phase == 'train'):
                output = model(images)
                loss = criterion(output, labels)
                preds = torch.argmax(output, 1)
              
                if phase == 'train':
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
            running_loss += loss.item()
            running_acc += (preds == labels).sum().item()
        
        epoch_loss = running_loss / len(imgs_ds[phase])
        epoch_acc = running_acc / len(imgs_ds[phase])
        print(f'{phase.capitalize()} Loss : {epoch_loss:.4f}, '
              f'{phase.capitalize()} Accuracy : {epoch_acc:.2%}')

        # Deep copy best model
        if phase == 'valid'and epoch_acc > best_acc:
            best_acc = epoch_acc
            best_model_weights = copy.deepcopy(model.state_dict())

        # Update losses and accuracies
        history['loss'][phase].append(epoch_loss)
        history['accuracy'][phase].append(epoch_acc)

    print()
time_elapsed = time.time() - start
print(f'Training complete in : {time_elapsed // 60}m and {time_elapsed % 60:.0f}s')
print(f'Best validation accuracy : {best_acc:.2%}')
model.load_state_dict(best_model_weights)
model.load_state_dict(best_model_weights)
model.layer1[0].conv1.weight.data[0, 0], \
model.layer1[0].conv1.weight.data[0, 1]
model.layer1[0].conv1.weight.grad, \
model.layer1[0].conv1.weight.grad<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13561582925744775, 'fraction_numerical': 0.01956425077812361, 'mean_word_length': 2.461716044632551, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '305837', 'license': 'Apache-2.0', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2858, 'n_words': 587, 'path': 'deep-learning-with-pytorch/notebooks/Bees-vs-Ants Experiments.ipynb', 'repo_name': 'ImadDabbura/dl-zoo', 'size': '8996'}