<jupyter_start><jupyter_text>Load the data Modelling (BaseLine) Modelling (Testing) Modelling (Final)<jupyter_code>import os
import cv2
import torch
import numpy as np
BATCH_SIZE = 32
from tqdm import tqdm
def load_data(img_size=112):
    data = []
    index = -1
    labels = {}
    for directory in os.listdir('./data/')[:37]:
        index += 1
        labels[f'./data/{directory}/'] = [index,-1]
    for label in tqdm(labels):
        for file in os.listdir(label):
            filepath = label + file
            img = cv2.imread(filepath)
            img = cv2.resize(img,(img_size,img_size))
#             img = img / 255.0
            data.append([
                np.array(img),
                labels[label][0]
            ])
            labels[label][1] += 1
    for _ in range(12):
        np.random.shuffle(data)
    print(len(data))
#     np.save('./data.npy',data)
    return data
import torch
def other_loading_data_proccess(data):
    print(len(data))
    X = []
    y = []
    print('going through the data..')
    for d in data:
        X.append(d[0])
        y.append(d[1])
    print('splitting the data')
    VAL_SPLIT = 0.25
    VAL_SPLIT = len(X)*VAL_SPLIT
    VAL_SPLIT = int(VAL_SPLIT)
    X_train = X[:-VAL_SPLIT]
    y_train = y[:-VAL_SPLIT]
    X_test = X[-VAL_SPLIT:]
    y_test = y[-VAL_SPLIT:]
    print('turning data to tensors')
    X_train = torch.from_numpy(np.array(X_train))
    print(len(X_train))
    y_train = torch.from_numpy(np.array(y_train))
    print(len(y_train))
    X_test = torch.from_numpy(np.array(X_test))
    print(len(X_test))
    y_test = torch.from_numpy(np.array(y_test))
    print(len(y_test))
    return [X_train,X_test,y_train,y_test]
REBUILD_DATA = True
if REBUILD_DATA:
    data = load_data()
    np.random.shuffle(data)
    X_train,X_test,y_train,y_test = other_loading_data_proccess(data)
import torch
import torch.nn as nn
from torchvision import transforms
import matplotlib.pyplot as plt
transform_train = transforms.Compose([
    transforms.RandomCrop(112, padding=2),
    transforms.RandomHorizontalFlip(),
    transforms.CenterCrop(112),
    transforms.RandomHorizontalFlip(),
    transforms.ColorJitter(0.25,0.25),
    transforms.ToTensor(),
    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
from PIL import Image
X_train_new = []
import random
for index in range(len(X_train)):
    testing = X_train[index]
    testing = np.array(testing)
    testing = Image.fromarray(testing)
    X_train_new.append(np.array(transform_train(testing)))
X_train_new = np.array(X_train_new)
X_train_new = X_train_new.astype(int)
X_train = torch.from_numpy(X_train_new)
X_train.shape<jupyter_output><empty_output><jupyter_text>Modelling BaseLine<jupyter_code>import torch
import torch.nn as nn
from torchvision import models
device = torch.device('cuda')
model = models.resnet18(pretrained=True).to(device)
model = model.to(device)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 37)
optimizer = torch.optim.SGD(model.parameters(),lr=0.1)
criterion = nn.CrossEntropyLoss()
EPOCHS = 12
BATCH_SIZE = 32
from tqdm import tqdm
loss_logs = []
for _ in tqdm(range(EPOCHS)):
    for i in range(0,len(X_train),BATCH_SIZE):
        X_batch = X_train[i:i+BATCH_SIZE].view(-1,3,112,112).to(device)
        y_batch = y_train[i:i+BATCH_SIZE].to(device)
        model = model.to(device)
        preds = model(X_batch.float().to(device))
        preds = preds.to(device)
        loss = criterion(preds,y_batch)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        loss_logs.append(loss.item())
plt.plot(loss_logs)
y_batch.shape
preds.shape
preds_new = []
for pred in preds:
    preds_new.append(int(torch.argmax(torch.round(pred))))
def get_loss(criterion,y,model,X):
    model.to('cpu')
    preds = model(X.view(-1,1,112,112).to('cpu').float())
    preds.to('cpu')
    loss = criterion(preds,torch.tensor(y,dtype=torch.long).to('cpu'))
    loss.backward()
    return loss.item()
def test(net,X,y):
    device = 'cpu'
    net.to(device)
    correct = 0
    total = 0
    net.eval()
    with torch.no_grad():
        for i in range(len(X)):
            real_class = torch.argmax(y[i]).to(device)
            net_out = net(X[i].view(-1,1,112,112).to(device).float())
            net_out = net_out[0]
            predictied_class = torch.argmax(net_out)
            if predictied_class == real_class:
                correct += 1
            total += 1
    net.train()
    net.to('cuda')
    return round(correct/total,3)
PROJECT_NAME = 'fruits-37'
import wandb
optimizers = [torch.optim.Adam,torch.optim.AdamW,torch.optim.Adamax,torch.optim.SGD]
for optimizer in optimizers:
    model = models.resnet18(pretrained=True).to(device)
    model = model.to(device)
    num_ftrs = model.fc.in_features
    model.fc = nn.Linear(num_ftrs, 37)
    optimizer = optimizer(model.parameters(),lr=0.1)
    criterion = nn.CrossEntropyLoss()
    wandb.init(project=PROJECT_NAME,name=f'optimizer-{optimizer}')
    for _ in tqdm(range(EPOCHS)):
        for i in range(0,len(X_train),BATCH_SIZE):
            X_batch = X_train[i:i+BATCH_SIZE].view(-1,3,112,112).to(device)
            y_batch = y_train[i:i+BATCH_SIZE].to(device)
            model = model.to(device)
            preds = model(X_batch.float().to(device))
            preds = preds.to(device)
            loss = criterion(preds,y_batch)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        wandb.log({'loss':loss.item(),'accuracy':test(model,X_train,y_train),'val_accuracy':test(model,X_test.view(-1,3,112,112),y_test),'val_loss':get_loss(criterion,y_test,model,X_test)})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14047994394815205, 'fraction_numerical': 0.023822035382729024, 'mean_word_length': 3.2421991084695394, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '15301', 'license': 'Apache-2.0', 'n_tokens_mistral': 2194, 'n_tokens_neox': 2105, 'n_words': 343, 'path': 'wandb/run-20210521_011115-3uji91xr/tmp/code/00-main.ipynb', 'repo_name': 'Programmer-RD-AI/fruits-37', 'size': '5709'}