<jupyter_start><jupyter_text>Imports, Mount Drive<jupyter_code># I took this kaggle problem
# I tried to solve it in pytorch
# This network here does not learn the task correctly. Try to make it work

#https://www.kaggle.com/karanjakhar/facial-keypoint-detection
#https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html

import pandas as pd
from google.colab import drive
import numpy as np
import matplotlib.pyplot as plt
import random
import torch
drive.mount('/content/drive/', force_remount=False)
%cd '/content/drive/My Drive/Applied 2020.06.06/Week 1 Materials'
!ls<jupyter_output>/content/drive/My Drive/Applied 2020.06.06/Week 1 Materials
'1- mnist_classification_dense_tensorflow.ipynb'   GANExample.ipynb
'2- mnist_classification_conv_kerasF.ipynb'	   Programme.txt
'3- cifar_classification_conv_kerasS.ipynb'	   training.csv
'4- kaggle_localisation_conv_keras.ipynb'	   training.zip
'4- kaggle_localisation_conv_pytorch.ipynb'	  'Week 1.gslides'
'5- segmentation_conv_pytorch.ipynb'		  'z1- Pytorch.ipynb'
'Applied AI Week 1.gdoc'<jupyter_text>Data Examination<jupyter_code>#!unzip training.zip
#!ls
data = pd.read_csv('training.csv') 
print(data.head())
print(data.isnull().any().value_counts())
data.fillna(method = 'ffill',inplace = True)
print(data.isnull().any().value_counts())
datalen = 7049
trainlen = 7000<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>pixel_list = []
for i in range(0,datalen):
    row = data['Image'][i].split(' ')
    pixel = ['0' if x == '' else x for x in row]
    pixel_list.append(pixel)
# Pytorch takes channels in the second dimension. For that, I swap axes (dimensions)
input_tensor = np.array(pixel_list,dtype = 'float')
print(np.shape(input_tensor))
input_tensor = input_tensor.reshape(-1,96,96,1)
input_tensor = np.swapaxes(input_tensor,2,3)
input_tensor = np.swapaxes(input_tensor,1,2)
print(np.shape(input_tensor))
labels = data.drop('Image',axis = 1)

label_list = []
for i in range(0,datalen):
    label = labels.iloc[i,:]
    label_list.append(label)
label_tensor = np.array(label_list,dtype = 'float')
from matplotlib.pyplot import Circle

index = random.randint(0,1000)

fig, ax = plt.subplots(1)
ax.set_aspect('equal')

ax.imshow(input_tensor[index].reshape(96,96),cmap='gray')

for xx, yy in label_tensor[index].reshape((15,2)):
    circ = Circle((xx,yy),2,color='red')
    ax.add_patch(circ)
inp_and_label = []
for i in range(trainlen):
   inp_and_label.append([input_tensor[i], label_tensor[i]])

# we use Dataloader objects in pytorch to easily iterate on our dataset while performing training loops
trainloader = torch.utils.data.DataLoader(inp_and_label, shuffle=True, batch_size=500)
i1, l1 = next(iter(trainloader))
print(i1.shape,l1.shape)<jupyter_output>torch.Size([500, 1, 96, 96]) torch.Size([500, 30])<jupyter_text>Network<jupyter_code>import torch.nn as nn
import torch.nn.functional as F

# we write our networks as class instances. dont forget to inherit from nn.Module
class Net(nn.Module):
    # we always need an init method to define our output matrices (similar to nodes in graph)
    # we need batch norms but i cannot make it run with batch norms
    def __init__(self):
        super(Net, self).__init__()
        
        self.conv1 = nn.Conv2d(1, 32, 5)
        self.conv1_bn = nn.BatchNorm2d(32)
        self.pool = nn.MaxPool2d(2,2)

        self.conv2 = nn.Conv2d(32, 64, 5)
        self.conv2_bn = nn.BatchNorm2d(64)

        self.conv3 = nn.Conv2d(64, 128, 5)
        self.conv3_bn = nn.BatchNorm2d(128)

        self.conv4 = nn.Conv2d(128, 256, 5)
        self.conv4_bn = nn.BatchNorm2d(256)


                                
        
        self.fc1 = nn.Linear(512 * 34 * 34, 256)
        self.fc1_bn = nn.BatchNorm1d(256)
        self.fc2 = nn.Linear(256, 84)
        self.fc2_bn = nn.BatchNorm1d(84)
        self.fc3 = nn.Linear(84, 30)
        
    # we always need an forward method to draw our computational graph (similar to completing the graph with edges)
    def forward(self, x):
        x = self.pool(F.sigmoid(self.conv1_bn(self.conv1(x))))
        x = F.sigmoid(self.conv2_bn(self.conv2(x)))
        x = F.sigmoid(self.conv3_bn(self.conv3(x)))
        x = F.sigmoid(self.conv4_bn(self.conv4(x)))

        # vectorize (flatten)
        x = x.reshape(-1, 512 * 25 * 25)

        #x = F.sigmoid(self.fc1_bn(self.fc1(x)))
        #x = F.sigmoid(self.fc2_bn(self.fc2(x)))

        x = F.sigmoid(self.fc1(x))
        x = F.sigmoid(self.fc2(x))
        x = self.fc3(x)
        return x


net = Net()
# i couldnt run the previous cell without errors, so i am omitting batch norms.
# but now, it will not learn correctly
import torch.nn as nn
import torch.nn.functional as F


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(32, 64, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.fc1 = nn.Linear(64 * 21 * 21, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 30)

    def forward(self, x):
        x = self.pool(F.sigmoid(self.conv1(x)))
        x = self.pool(F.sigmoid(self.conv2(x)))
        x = x.reshape(-1, 64 * 21 * 21)
        x = F.sigmoid(self.fc1(x))
        x = F.sigmoid(self.fc2(x))
        x = self.fc3(x)
        return x


net = Net()
import torch.optim as optim

criterion = nn.MSELoss()
optimizer = optim.SGD(net.parameters(), lr=0.0001, momentum=0.9)
net = net.float()
#we get info on our gpu, put it in the variable "device"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Assuming that we are on a CUDA machine, this should print a CUDA device:

print(device)

#we carry our model into gpu
net.to(device)


# our training loop

for epoch in range(10):  # loop over the dataset multiple times

    running_loss = 0.0

    # here we use the dataloader object. it performs .next() operation in each iteration of the loop
    # we also group our batches with numbers. we do this with enumerate. we do this so we can know in which batch we are 
    for i, data in enumerate(trainloader, start = 0):
        # get the inputs; data is a list of [inputs, labels]

        #inputs, labels = data
        inputs, labels = data[0].float().to(device), data[1].float().to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 100 == 0:    # print every 2000 mini-batches
            print('Epoch %d Loss: %.3f' %
                  (epoch + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')
test = []
for i in range(trainlen,datalen):
   test.append([input_tensor[i], label_tensor[i]])

testloader = torch.utils.data.DataLoader(test, shuffle=True, batch_size=500)
i1, l1 = next(iter(testloader))
print(i1.shape,l1.shape)
# In pytorch, dont forget to perform testing evaluations with no_grad. Or else, it 
# fills the memory of the gpu and it eventually makes you restart the computer.

# If we train enough, we see that our network makes almost identical predictions for each image
# this is not good.
# We may try to solve this with changing the model structure, making it deeper, adding batch norm, etc.

from matplotlib.pyplot import Circle
randomsayi = random.randint(0,49)

#inputs = torch.tensor(input_tensor[randomsayi:randomsayi+2].reshape((2,1,96,96))).float().to(device)
#labels = torch.tensor(label_tensor[randomsayi:randomsayi+2]).float().to(device)

print(np.shape(inputs))
preds = net(inputs).cpu()


fig, ax = plt.subplots(1)
ax.set_aspect('equal')

ax.imshow(inputs[randomsayi].cpu().view((96,96)),cmap = 'gray')

for xx, yy in preds[randomsayi].reshape((15,2)):
    circ = Circle((xx,yy),2,color='red')
    ax.add_patch(circ)

from matplotlib.pyplot import Circle
randomsayi = random.randint(0,49)

#inputs = torch.tensor(input_tensor[randomsayi:randomsayi+2].reshape((2,1,96,96))).float().to(device)
#labels = torch.tensor(label_tensor[randomsayi:randomsayi+2]).float().to(device)

print(np.shape(inputs))
preds = net(inputs).cpu()


fig, ax = plt.subplots(1)
ax.set_aspect('equal')

ax.imshow(inputs[randomsayi].cpu().view((96,96)),cmap = 'gray')

for xx, yy in preds[randomsayi].reshape((15,2)):
    circ = Circle((xx,yy),2,color='red')
    ax.add_patch(circ)

fig, ax = plt.subplots(1)
ax.set_aspect('equal')

ax.imshow(inputs[randomsayi].cpu().view((96,96)),cmap = 'gray')

for xx, yy in labels[randomsayi].reshape((15,2)):
    circ = Circle((xx,yy),2,color='red')
    ax.add_patch(circ)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13452557238865476, 'fraction_numerical': 0.037475794509625245, 'mean_word_length': 3.6210526315789475, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '534890', 'license': 'MIT', 'n_tokens_mistral': 3381, 'n_tokens_neox': 3158, 'n_words': 760, 'path': 'week1/5- kaggle_localisation_conv_pytorch.ipynb', 'repo_name': 'onurboyar/Applied-AI-Study-Group-2020-June', 'size': '8779'}