<jupyter_start><jupyter_text>Deep learning Approach  Imports<jupyter_code>import numpy as np 
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torch import optim
import matplotlib.pyplot as plt
torch.manual_seed(200)
import warnings<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>def extract_data(torch_subset):
    df = torch_subset.dataset.iloc[torch_subset.indices]
    x = torch.from_numpy(df.values[ : , 1:]).reshape(-1, 28, 28).to(torch.float)
    y = torch.from_numpy(df.values[:, 0]).to(torch.long)
    
    return x, y
    
    
def split(dataframe):
    val_size = len(dataframe)//6
    train_size = len(dataframe) - val_size
    
    train, valid = torch.utils.data.random_split(dataframe, [train_size, val_size])

    return extract_data(train), extract_data(valid)

def show_img(X, y):
    fig = plt.figure(figsize=(20, 5))
    for i in range(10):
        ax = fig.add_subplot(2, 5, i+1, xticks=[], yticks=[])
        ax.imshow(X[i].view(28, 28))
        ax.set_title(str(y[i].item()))<jupyter_output><empty_output><jupyter_text>Custom Dataset<jupyter_code>class MNISTDS(torch.utils.data.Dataset):
    def __init__(self, images, targets, transforms=None):
        self.images = images.unsqueeze(1) / 255
        self.targets = targets
        self.transforms = transforms
        
    def __len__(self):
        return len(self.targets)
    
    def __getitem__(self, i):
        if self.transforms:
            x = self.transforms(self.images[i])
        else:
            x = self.images[i]
        
        sample = {'image': x, 'target': self.targets[i]}
        return x, self.targets[i]<jupyter_output><empty_output><jupyter_text>Load data and transform to torch tesnors<jupyter_code>train_df = pd.read_csv('data/train.csv')
test_df = pd.read_csv('data/test.csv')
(X_train, y_train), (X_val, y_val) = split(train_df)
X_test = torch.Tensor(test_df.values).reshape(-1, 28, 28)<jupyter_output><empty_output><jupyter_text>Explore<jupyter_code>show_img(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Data Augmentation<jupyter_code>import torchvision.transforms as transforms

transform = torch.nn.Sequential(
#     transforms.GaussianBlur(3, sigma=(0.001, 0.3)),
    transforms.RandomAffine(degrees=10, fillcolor=None),
)
scripted_transforms = torch.jit.script(transform)<jupyter_output><empty_output><jupyter_text>View augmented data<jupyter_code>show_img(scripted_transforms(X_train), y_train)<jupyter_output><empty_output><jupyter_text>Create dataset and dataloader<jupyter_code>trainset = MNISTDS(X_train, y_train)
validset = MNISTDS(X_val, y_val)
batch_size = 100
train_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(validset, batch_size=batch_size, shuffle=True)<jupyter_output><empty_output><jupyter_text>CNN<jupyter_code>class Classifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1) 
        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)
        self.conv3 = nn.Conv2d(32, 64, kernel_size=4, padding=1)
        self.conv4 = nn.Conv2d(64, 128, kernel_size=4, padding=1)
        
        self.pool = nn.MaxPool2d(2, 2)
        
        self.fc1 = nn.Linear(128, 512) 
        self.fc2 = nn.Linear(512, 10)
        
        self.dropout = nn.Dropout(p=0.30)
        
    def forward(self, x):
        x = x.view(x.shape[0], 1, 28, 28)
      
        x = self.conv1(x)
        x = F.relu(x)
        x = self.pool(x)
   
        x = self.conv2(x)
        x = F.relu(x)
        x = self.pool(x)
        
        x = self.conv3(x)
        x = F.relu(x)
        x = self.pool(x)
        
        x = self.conv4(x)
        x = F.relu(x)
        x = self.pool(x)
    
        x = self.dropout(x)
        x = x.view(-1, 128)
        x = self.dropout(x)
        
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout(x)
        
        x = self.fc2(x)
        
        return x
    
model = Classifier()<jupyter_output><empty_output><jupyter_text>Use GPU<jupyter_code>use_cuda = torch.cuda.is_available()

# move model to GPU if CUDA is available
if use_cuda:
    model = model.cuda()
    print("Working with cuda")<jupyter_output>Working with cuda<jupyter_text>Train and validate<jupyter_code>loss_function = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.35)
epoch = 30
train_losses, valid_losses = [], []
valid_loss_min = np.Inf

for i in range(epoch):
    tot_train_loss = 0
    
    model.train()
    for images, labels in train_loader:
        if use_cuda:
                images, labels = images.cuda(), labels.cuda()
             
        pred = model(scripted_transforms(images))
        loss = loss_function(pred, labels)
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        tot_train_loss += loss.item()
        
    else:
        tot_valid_loss = 0
        valid_correct = 0
        
        model.eval()
        with torch.no_grad():
        
            for images, labels in valid_loader:
                if use_cuda:
                    images, labels = images.cuda(), labels.cuda()
                    
                pred = model(scripted_transforms(images))
                loss = loss_function(pred, labels)
                tot_valid_loss += loss.item()
        
        train_loss = tot_valid_loss / len(train_loader.dataset)
        valid_loss = tot_valid_loss / len(valid_loader.dataset)
        
        train_losses.append(train_loss)
        valid_losses.append(valid_loss)
        
        print("Epoch: {}/{}.. ".format(i+1, epoch),
              "Training Loss: {:.3f}.. ".format(train_loss),
              "Test Loss: {:.3f}.. ".format(valid_loss),
              )
        
        if valid_loss <= valid_loss_min:
            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
            valid_loss_min,
            valid_loss))
            torch.save(model.state_dict(), 'model.pt')
            valid_loss_min = valid_loss<jupyter_output>Epoch: 1/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Validation loss decreased (inf --> 0.001363).  Saving model ...
Epoch: 2/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Validation loss decreased (0.001363 --> 0.000824).  Saving model ...
Epoch: 3/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Validation loss decreased (0.000824 --> 0.000626).  Saving model ...
Epoch: 4/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Validation loss decreased (0.000626 --> 0.000582).  Saving model ...
Epoch: 5/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Epoch: 6/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Epoch: 7/30..  Training Loss: 0.000..  Test Loss: 0.000.. 
Validation loss decreased (0.000582 --> 0.000446).  Saving model ...
Epoch: 8/30..  Training Loss: 0.000..  Test Loss: 0.001.. 
Epoch: 9/30..  Training Loss: 0.000..  Test Loss: 0.000.. 
Validation loss decreased (0.000446 --> 0.000420).  Saving model ...
Epoch: 10/30..  Training Loss: 0.000..  Test Loss: 0.000.. 
[...]<jupyter_text>Train valid loss<jupyter_code>plt.plot(train_losses, label='Train Loss')
plt.plot(valid_losses, label='Valid Loss')<jupyter_output><empty_output><jupyter_text>Load best model<jupyter_code>model = Classifier()
model.load_state_dict(torch.load('model.pt'))

if use_cuda:
    model = model.cuda()
    print("Working with cuda")<jupyter_output>Working with cuda<jupyter_text>Predict test data<jupyter_code>class TestDataset(torch.utils.data.Dataset):
    def __init__(self, images):
        self.images = images.unsqueeze(1).cuda() / 255

    def __len__(self):
        return len(self.images)

    def __getitem__(self, i):
        return self.images[i]
    
    
testset = TestDataset(X_test)
test_loader = DataLoader(testset, batch_size=100, shuffle=False, num_workers=0)
preds_class = np.array([], dtype=int)

with torch.no_grad():
    for images in test_loader:
        preds = model(images.cuda()).cpu() # only cpu tensor can be converted to the numpy array
        preds_class = np.append(preds_class, preds.argmax(dim=1))<jupyter_output><empty_output><jupyter_text>Save as Pandas<jupyter_code>df = pd.DataFrame({
    # notice that the sample submission iterates id's from 1
    'ImageId': np.arange(1, len(preds_class) + 1),
    'Label': preds_class
})
# set the ImageId row as an index
df = df.set_index('ImageId')
df
df.to_csv('submission.csv')
len(df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13226710249824888, 'fraction_numerical': 0.037823955171608685, 'mean_word_length': 2.8279714030384273, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '421793', 'license': 'MIT', 'n_tokens_mistral': 3288, 'n_tokens_neox': 3037, 'n_words': 601, 'path': '.ipynb_checkpoints/handwritten-digit-recognizer-checkpoint.ipynb', 'repo_name': 'NinaM31/handDigitRecognizer', 'size': '8566'}