<jupyter_start><jupyter_text>Image classification test 1  Load dataset<jupyter_code>!git clone https://github.com/Lindronics/honours_project_data<jupyter_output>Cloning into 'honours_project_data'...
remote: Enumerating objects: 128, done.[K
remote: Counting objects:   0% (1/128)[K
remote: Counting objects:   1% (2/128)[K
remote: Counting objects:   2% (3/128)[K
remote: Counting objects:   3% (4/128)[K
remote: Counting objects:   4% (6/128)[K
remote: Counting objects:   5% (7/128)[K
remote: Counting objects:   6% (8/128)[K
remote: Counting objects:   7% (9/128)[K
remote: Counting objects:   8% (11/128)[K
remote: Counting objects:   9% (12/128)[K
remote: Counting objects:  10% (13/128)[K
remote: Counting objects:  11% (15/128)[K
remote: Counting objects:  12% (16/128)[K
remote: Counting objects:  13% (17/128)[K
remote: Counting objects:  14% (18/128)[K
remote: Counting objects:  15% (20/128)[K
remote: Counting objects:  16% (21/128)[K
remote: Counting objects:  17% (22/128)[K
remote: Counting objects:  18% (24/128)[K
remote: Counting objects:  19% (25/128)[K
remote: Counting objects:  20% (26/128)[K
remote: Counting object[...]<jupyter_text>Data loaderLoads FIR data<jupyter_code>import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data.sampler import SubsetRandomSampler

import torchvision
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = [15, 10]
from os import listdir
from os import path
import json
import cv2

class HandsDataset(torch.utils.data.Dataset):
    
    def __init__(self, dir):
        with open(path.join(dir, "labels.json"), "r") as f:
            labels = json.load(f)
        self.files, self.labels = zip(*labels.items())
        self.dir = dir

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_path = path.join(self.dir, "thermal", self.files[idx])

        img = cv2.imread(img_path) / 128 - 1
        img = cv2.resize(img, (128, 128))
        img = np.mean(img, axis=2)[None, ...]

        return {
            "label": self.labels[idx],
            "image": img
        }
        
dataset = HandsDataset("honours_project_data/hands")
dataset[10]
# Train test split

dataset_size = len(dataset)
indices = list(range(dataset_size))
split = int(0.2 * dataset_size)

# Shuffle dataset
np.random.seed(100)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]

print(train_indices)
print(val_indices)

# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)

train_loader = torch.utils.data.DataLoader(dataset, batch_size=4, sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(dataset, batch_size=4, sampler=valid_sampler)
def imshow(img):
    img = img / 2 + 0.5
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

def show_batch(batch):  
    img, labels = batch["image"], batch["label"]
    imshow(torchvision.utils.make_grid(img))
    print(labels)

dataiter = iter(train_loader)
show_batch(dataiter.next())<jupyter_output><empty_output><jupyter_text>The model<jupyter_code>class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16*29*29, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 4)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16*29*29)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()
net = net.float()
import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

for epoch in range(70):
    print(f"Starting epoch {epoch}")
    running_loss = 0.0
    for i, data in enumerate(train_loader, 0):
        # get the inputs; data is a list of [inputs, labels]
        imgs, labels = data["image"].float(), data["label"]
        
        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(imgs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 100 == 99:    # print every 100 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 100))
            running_loss = 0.0

print('Finished Training')
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix

all_outputs = []
all_labels = []
for i, data in enumerate(validation_loader, 1000):
    inputs, labels = data["image"].float(), data["label"]
    outputs = net(inputs)
    _, predicted = torch.max(outputs, 1)
    all_outputs += predicted.tolist()
    all_labels += labels.tolist()
print(classification_report(all_labels, all_outputs))
cm = confusion_matrix(all_labels, all_outputs)

fig, ax = plt.subplots()
im = ax.imshow(cm, interpolation='nearest')
ax.figure.colorbar(im, ax=ax)
ax.set(xticks=np.arange(cm.shape[1]),
    yticks=np.arange(cm.shape[0]),
    xticklabels=classes, yticklabels=classes,
    ylabel='True label',
    xlabel='Predicted label')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14178302900107412, 'fraction_numerical': 0.04421768707482993, 'mean_word_length': 3.4731785428342676, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '196522', 'license': 'MIT', 'n_tokens_mistral': 2130, 'n_tokens_neox': 1952, 'n_words': 456, 'path': 'notebooks/classification/hands_classification.ipynb', 'repo_name': 'Lindronics/honours_project_dissertation', 'size': '5586'}