<jupyter_start><jupyter_text>try implement cnn model from pytorch  import lib<jupyter_code>%matplotlib inline
from pathlib import Path
import requests
import gzip
import numpy as np
import math
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader

from sklearn.metrics import accuracy_score<jupyter_output><empty_output><jupyter_text>the original model cnn<jupyter_code>class mycnn(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=4, kernel_size=(2, 2), stride=2, padding=0)
    self.pool1 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv2 = nn.Conv2d(in_channels=4, out_channels=8, kernel_size=(2, 2), stride=1, padding=0)
    self.pool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.fc_layer=nn.Linear(72,10)

  def forward(self, xb):
    xb = self.conv1(xb)
    xb = self.pool1(xb)
    xb = F.relu(xb)

    xb = self.conv2(xb)
    xb = self.pool2(xb)
    xb = F.relu(xb)

    xb = xb.view(-1,72)  # reshape

    xb = self.fc_layer(xb)
    
    return xb
def train_model(model, x_train, y_train, x_test, y_test, epochs=250, batch_size=64, lr=0.01, weight_decay=0):
  # data
  train_dataset = TensorDataset(x_train, y_train)
  train_data_loader = DataLoader(train_dataset, batch_size=batch_size)
  
  # loss function
  loss_func = F.cross_entropy

  # optimizer
  optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay=weight_decay)

  # training loop
  for epoch in range(epochs):
    for xb, yb in train_data_loader:
      
      pred = model(xb)
      loss = loss_func(pred, yb)

      loss.backward()
      optimizer.step()
      optimizer.zero_grad()

    y_pred = model(x_test)
    acc = accuracy_score(torch.argmax(y_pred, dim=1).cpu().detach().numpy(), y_test.cpu())
    
    print('Loss at epoch %d : %f, test acc: %f' % (epoch, loss, acc))<jupyter_output><empty_output><jupyter_text>prepare data<jupyter_code>DATA_PATH = Path("data")
PATH = DATA_PATH / "mnist"

PATH.mkdir(parents=True, exist_ok=True)

def get_data(url, file_name, offset):
  content = requests.get(url).content
  (PATH / file_name).open("wb").write(content)
  with gzip.open((PATH / file_name).as_posix(), "rb") as f:
    data = np.frombuffer(f.read(), dtype=np.uint8, offset=offset)
  return data

y_train = get_data("http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz", "train-labels", 8).astype('int64')
x_train = get_data("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz", "train-images", 16).reshape(len(y_train), 784).astype('float32')
y_test = get_data("http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz", "test-labels", 8).astype('int64')
x_test = get_data("http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz", "test-images", 16).reshape(len(y_test), 784).astype('float32')
x_train, y_train, x_test, y_test = map(
    torch.tensor, (x_train, y_train, x_test, y_test)
)
n = x_train.shape[0]<jupyter_output><empty_output><jupyter_text>cuda available<jupyter_code>x_test=x_test.cuda()
x_train=x_train.cuda()
y_test=y_test.cuda()
y_train=y_train.cuda()
mycnn1= mycnn()
mycnn1.cuda()
train_model(mycnn1, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.0001)
train_model(mycnn1, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.0001)
train_model(mycnn1, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.0001)<jupyter_output>Loss at epoch 0 : 0.199632, test acc: 0.890600
Loss at epoch 1 : 0.193213, test acc: 0.891800
Loss at epoch 2 : 0.188237, test acc: 0.893400
Loss at epoch 3 : 0.184487, test acc: 0.894100
Loss at epoch 4 : 0.180569, test acc: 0.894600
Loss at epoch 5 : 0.177181, test acc: 0.895600
Loss at epoch 6 : 0.173735, test acc: 0.896300
Loss at epoch 7 : 0.170623, test acc: 0.896700
Loss at epoch 8 : 0.168641, test acc: 0.897700
Loss at epoch 9 : 0.167001, test acc: 0.898600
Loss at epoch 10 : 0.166416, test acc: 0.899200
Loss at epoch 11 : 0.165784, test acc: 0.899100
Loss at epoch 12 : 0.165009, test acc: 0.899700
Loss at epoch 13 : 0.165316, test acc: 0.900300
Loss at epoch 14 : 0.166102, test acc: 0.900800
Loss at epoch 15 : 0.166592, test acc: 0.901100
Loss at epoch 16 : 0.167382, test acc: 0.902400
Loss at epoch 17 : 0.167557, test acc: 0.902800
Loss at epoch 18 : 0.168183, test acc: 0.903500
Loss at epoch 19 : 0.167581, test acc: 0.904400
Loss at epoch 20 : 0.167333, test acc: 0.904800
Lo[...]<jupyter_text>try new cnn model<jupyter_code>class mycnnNew(nn.Module):
    def __init__(self):
      super().__init__()
        
      self.features = nn.Sequential(
          nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
          nn.BatchNorm2d(32),
          nn.ReLU(inplace=True),
          nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),
          nn.BatchNorm2d(32),
          nn.ReLU(inplace=True),
          nn.MaxPool2d(kernel_size=2, stride=2),
          nn.Conv2d(32, 64, kernel_size=3, padding=1),
          nn.BatchNorm2d(64),
          nn.ReLU(inplace=True),
          nn.Conv2d(64, 64, kernel_size=3, padding=1),
          nn.BatchNorm2d(64),
          nn.ReLU(inplace=True),
          nn.MaxPool2d(kernel_size=2, stride=2)
      )
        
      self.classifier = nn.Sequential(
          nn.Dropout(p = 0.5),
          nn.Linear(64 * 7 * 7, 512),
          nn.BatchNorm1d(512),
          nn.ReLU(inplace=True),
          nn.Dropout(p = 0.5),
          nn.Linear(512, 512),
          nn.BatchNorm1d(512),
          nn.ReLU(inplace=True),
          nn.Dropout(p = 0.5),
          nn.Linear(512, 10),
      )
        
      for m in self.features.children():
          if isinstance(m, nn.Conv2d):
              n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
              m.weight.data.normal_(0, math.sqrt(2. / n))
          elif isinstance(m, nn.BatchNorm2d):
              m.weight.data.fill_(1)
              m.bias.data.zero_()
      
      for m in self.classifier.children():
          if isinstance(m, nn.Linear):
              nn.init.xavier_uniform(m.weight)
          elif isinstance(m, nn.BatchNorm1d):
              m.weight.data.fill_(1)
              m.bias.data.zero_()
                

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        
        return x     
mycnn2= mycnnNew()
mycnn2.cuda()
def train_model(model, x_train, y_train, x_test, y_test, epochs=250, batch_size=256, lr=0.01, weight_decay=0):
  # data
  train_dataset = TensorDataset(x_train, y_train)
  train_data_loader = DataLoader(train_dataset, batch_size=batch_size)
  
  # loss function
  loss_func = F.cross_entropy

  # optimizer
  optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)

  # training loop
  for epoch in range(epochs):
    for xb, yb in train_data_loader:
      
      pred = model(xb)
      loss = loss_func(pred, yb)

      loss.backward()
      optimizer.step()
      optimizer.zero_grad()

    y_pred = model(x_test)
    acc = accuracy_score(torch.argmax(y_pred, dim=1).cpu().detach().numpy(), y_test.cpu())
    
    print('Loss at epoch %d : %f, test acc: %f' % (epoch, loss, acc))
train_model(mycnn2, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.001)<jupyter_output>Loss at epoch 0 : 0.024664, test acc: 0.980600<jupyter_text>try new cnn model2 without fully connected layer<jupyter_code>class mycnnNew2(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=4, kernel_size=(2, 2), stride=2, padding=0)
    self.pool1 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv2 = nn.Conv2d(in_channels=4, out_channels=8, kernel_size=(2, 2), stride=1, padding=0)
    self.pool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv3 = nn.Conv2d(in_channels=8, out_channels=10, kernel_size=(2, 2), stride=1, padding=0)
    self.pool3 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

  def forward(self, xb):
    xb = self.conv1(xb)
    xb = self.pool1(xb)
    xb = F.relu(xb)

    xb = self.conv2(xb)
    xb = self.pool2(xb)
    xb = F.relu(xb)

    xb = self.conv3(xb)
    xb = self.pool3(xb)
    xb = F.relu(xb)

    xb = xb.view(-1,10)  # reshape
    
    return xb
mycnn2= mycnnNew2()
mycnn2.cuda()
train_model(mycnn2, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.001)<jupyter_output>Loss at epoch 0 : 1.294848, test acc: 0.657200
Loss at epoch 1 : 0.966768, test acc: 0.752900
Loss at epoch 2 : 0.588402, test acc: 0.796900
Loss at epoch 3 : 0.384869, test acc: 0.821300
Loss at epoch 4 : 0.299969, test acc: 0.827000
Loss at epoch 5 : 0.266450, test acc: 0.832100
Loss at epoch 6 : 0.253402, test acc: 0.835300
Loss at epoch 7 : 0.275465, test acc: 0.839900
Loss at epoch 8 : 0.303268, test acc: 0.842600
Loss at epoch 9 : 0.319065, test acc: 0.845300
Loss at epoch 10 : 0.313331, test acc: 0.848000
Loss at epoch 11 : 0.313610, test acc: 0.850300
Loss at epoch 12 : 0.297781, test acc: 0.852200
Loss at epoch 13 : 0.288426, test acc: 0.853600
Loss at epoch 14 : 0.289039, test acc: 0.853700
Loss at epoch 15 : 0.295336, test acc: 0.855900
Loss at epoch 16 : 0.294172, test acc: 0.856300
Loss at epoch 17 : 0.281977, test acc: 0.857600
Loss at epoch 18 : 0.279066, test acc: 0.856100
Loss at epoch 19 : 0.274937, test acc: 0.854400
Loss at epoch 20 : 0.266525, test acc: 0.853600
Lo[...]<jupyter_text>try cnn new model3<jupyter_code>class mycnnNew3(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=8, kernel_size=(2, 2), stride=2, padding=0)
    self.pool1 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(2, 2), stride=1, padding=0)
    self.pool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv3 = nn.Conv2d(in_channels=16, out_channels=10, kernel_size=(2, 2), stride=1, padding=0)
    self.pool3 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

  def forward(self, xb):
    xb = self.conv1(xb)
    xb = self.pool1(xb)
    xb = F.relu(xb)

    xb = self.conv2(xb)
    xb = self.pool2(xb)
    xb = F.relu(xb)

    xb = self.conv3(xb)
    xb = self.pool3(xb)
    xb = F.relu(xb)

    xb = xb.view(-1,10)  # reshape
    
    return xb
mycnn3= mycnnNew3()
mycnn3.cuda()
train_model(mycnn3, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.001)<jupyter_output>Loss at epoch 0 : 0.841792, test acc: 0.650700
Loss at epoch 1 : 0.503979, test acc: 0.740000
Loss at epoch 2 : 0.276702, test acc: 0.809200
Loss at epoch 3 : 0.314182, test acc: 0.823900
Loss at epoch 4 : 0.286516, test acc: 0.828400
Loss at epoch 5 : 0.246339, test acc: 0.832300
Loss at epoch 6 : 0.231269, test acc: 0.838400
Loss at epoch 7 : 0.223435, test acc: 0.840200
Loss at epoch 8 : 0.221031, test acc: 0.840300
Loss at epoch 9 : 0.226826, test acc: 0.841300
Loss at epoch 10 : 0.216865, test acc: 0.840500
Loss at epoch 11 : 0.205376, test acc: 0.840200
Loss at epoch 12 : 0.185354, test acc: 0.846300
Loss at epoch 13 : 0.168543, test acc: 0.848000
Loss at epoch 14 : 0.159415, test acc: 0.848300
Loss at epoch 15 : 0.154316, test acc: 0.848200
Loss at epoch 16 : 0.150831, test acc: 0.849700
Loss at epoch 17 : 0.159984, test acc: 0.852600
Loss at epoch 18 : 0.154055, test acc: 0.853500
Loss at epoch 19 : 0.151885, test acc: 0.852900
Loss at epoch 20 : 0.154190, test acc: 0.853700
Lo[...]<jupyter_text>try new cnn model4 improve avgpool2d<jupyter_code>class mycnnNew4(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=8, kernel_size=(2, 2), stride=2, padding=0)
    self.pool1 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv2 = nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(2, 2), stride=1, padding=0)
    self.pool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv3 = nn.Conv2d(in_channels=16, out_channels=10, kernel_size=(2, 2), stride=1, padding=0)
    self.pool3 = nn.AvgPool2d(kernel_size=(2, 2), padding=0)

  def forward(self, xb):
    xb = self.conv1(xb)
    xb = self.pool1(xb)
    xb = F.leaky_relu(xb)

    xb = self.conv2(xb)
    xb = self.pool2(xb)
    xb = F.leaky_relu(xb)

    xb = self.conv3(xb)
    xb = self.pool3(xb)
    xb = F.leaky_relu(xb)

    xb = xb.view(-1,10)  # reshape
    
    return xb
mycnn4= mycnnNew4()
mycnn4.cuda()
train_model(mycnn4, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.001)<jupyter_output>Loss at epoch 0 : 2.079775, test acc: 0.289800
Loss at epoch 1 : 1.795111, test acc: 0.393100
Loss at epoch 2 : 1.538747, test acc: 0.531300
Loss at epoch 3 : 1.232358, test acc: 0.663300
Loss at epoch 4 : 0.631396, test acc: 0.821700
Loss at epoch 5 : 0.554915, test acc: 0.852800
Loss at epoch 6 : 0.466510, test acc: 0.871800
Loss at epoch 7 : 0.407404, test acc: 0.884600
Loss at epoch 8 : 0.372852, test acc: 0.894700
Loss at epoch 9 : 0.335485, test acc: 0.902800
Loss at epoch 10 : 0.316834, test acc: 0.909700
Loss at epoch 11 : 0.303526, test acc: 0.914100
Loss at epoch 12 : 0.288920, test acc: 0.917000
Loss at epoch 13 : 0.279811, test acc: 0.920100
Loss at epoch 14 : 0.268737, test acc: 0.924100
Loss at epoch 15 : 0.259610, test acc: 0.925100
Loss at epoch 16 : 0.249873, test acc: 0.927800
Loss at epoch 17 : 0.243584, test acc: 0.930400
Loss at epoch 18 : 0.238489, test acc: 0.931200
Loss at epoch 19 : 0.233559, test acc: 0.932400
Loss at epoch 20 : 0.227727, test acc: 0.933300
Lo[...]<jupyter_text>try cnn new model5<jupyter_code>class mycnnNew5(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=(2, 2), stride=2, padding=0)
    self.pool1 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv2 = nn.Conv2d(in_channels=2, out_channels=5, kernel_size=(2, 2), stride=1, padding=0)
    self.pool2 = nn.MaxPool2d(kernel_size=(2, 2), padding=0)

    self.conv3 = nn.Conv2d(in_channels=5, out_channels=10, kernel_size=(2, 2), stride=1, padding=1)
    self.pool3 = nn.AvgPool2d(kernel_size=(2, 2), padding=0)

  def forward(self, xb):
    xb = self.conv1(xb)
    xb = self.pool1(xb)
    xb = F.leaky_relu(xb)

    xb = self.conv2(xb)
    xb = self.pool2(xb)
    xb = F.leaky_relu(xb)

    xb = self.conv3(xb)
    xb = self.pool3(xb)
    xb = F.leaky_relu(xb)

    xb = xb.view(-1,10)  # reshape
    
    return xb
mycnn5 = mycnnNew5()
mycnn5.cuda()
train_model(mycnn5, x_train.view(-1,1,28,28), y_train, x_test.view(-1,1,28,28), y_test,epochs=30,lr=0.001)<jupyter_output><empty_output><jupyter_text>summarize<jupyter_code>list(mycnnNew.parameters(mycnn2))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1433571140126531, 'fraction_numerical': 0.12639655404495895, 'mean_word_length': 3.521911138161899, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '642384', 'license': 'MIT', 'n_tokens_mistral': 7335, 'n_tokens_neox': 6120, 'n_words': 1396, 'path': '20190813cnn.ipynb', 'repo_name': 'CharlesPoletowin/YCBS-273', 'size': '14858'}