<jupyter_start><jupyter_text>Image Preprocessing<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython.display import Image
import cv2
import os
from torchvision import transforms
import torch
import pickle
from random import shuffle
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from torchvision.utils import make_grid
import cv2
import numpy as np
import pandas as pd
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
from torch import optim
%matplotlib inline
(Image(filename='cropped.jpg'))<jupyter_output><empty_output><jupyter_text>Read in the image:<jupyter_code>filename = 'cropped.jpg'
img = cv2.imread(filename)
cv2.imshow("Image", img)
cv2.waitKey(0)
cv2.destroyAllWindows()
W = 1000
heigh, width, depth = img.shape
imgScale = W/width
newX, newY = img.shape[1]*imgScale, img.shape[0]*imgScale
new_img = cv2.resize(img, (int(newX), int(newY)))
cv2.imshow("Show by CV2", new_img)
cv2.waitKey(0)
cv2.imwrite('resize_white_img.png', new_img)
plt.imshow(new_img) 
invert_img = cv2.bitwise_not(new_img)
cv2.imshow("invert1", invert_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
new_img = (cv2.threshold(invert_img, 125, 255, cv2.THRESH_BINARY))
#new_img1 = cv2.threshold(new_img, 200, 255, cv2.THRESH_BINARY)
invert_img1 = new_img[1]
plt.imshow(invert_img1)
rows = [(15,125), (125,225), (235,335), (340,440), (455,555), (570,670), (680,780), (775,875), (890,990)]
columns = [(30,130), (130,230), (240,340), (355,455), (455,555), (565,665), (670,770), (800,900),(890,990)]
images_list = []
for unit in rows:
    for units in columns:
        images_list.append(invert_img1[unit[0]:unit[1], units[0]:units[1]])<jupyter_output><empty_output><jupyter_text>images_list = []for unit in rows:    for units in columns:        images_list.append(invert_img[unit[0]:unit[1], units[0]:units[1]])<jupyter_code>plt.imshow(images_list[0])<jupyter_output><empty_output><jupyter_text>To save the inverted and preprocessed images to local folder:<jupyter_code># for i in range(len(images_list)):
#    if i <= 9:
#        cv2.imwrite("0"+f"{i}_flip.png", images_list[i])
#    else:
#        cv2.imwrite(f"{i}_flip.png", images_list[i])
# for i in range(len(images_list)):
#    cv2.imwrite(f"0{i}_flip.png", images_list[i])

# cv2.imwrite("flip_img81.png", images_list[80])
# To pull in the images from the local folder, and prepare them to be passed onto the model:
final_images = []
for i in range(len(images_list)):
#    img_array = cv2.imread(os.path.join(IMG_DIR, images))
    img_array = cv2.cvtColor(images_list[i], cv2.COLOR_BGR2GRAY)
    resize_img = cv2.resize(img_array, (28,28))
    final_images.append(resize_img)
plt.imshow(final_images[78]) # 47, 57, 69, 
final_images[78].mean()
plt.imshow(final_images[78])
img_change = cv2.threshold(final_images[78], 130, 255, cv2.THRESH_BINARY)
#plt.imshow(img_change)<jupyter_output><empty_output><jupyter_text>How to save?<jupyter_code>plt.imshow(np.full_like(final_images[5], 0))
plt.imshow(final_images[59])<jupyter_output><empty_output><jupyter_text>Creating tuples<jupyter_code>plt.imshow(new_img)
plt.imshow(final_images[18])
copy_img71_1=[(np.copy(final_images[10]),1)for i in range(20)] # values is the image
copy_img71_2 = [(np.copy([final_images[47]]),1)for i in range(20)]
copy_img71_3 = [(np.copy([final_images[80]]), 1) for i in range(20)]
#opy_img61_4= [(np.copy([final_images[48]]), 1) for i in range(20)]
#opy_img61_5 = [(np.copy([final_images[68]]), 1) for i in range(20)]
copy_img71_1.extend(copy_img71_2)
copy_img71_1.extend(copy_img71_3)
# copy_img51_1.extend(copy_img51_4)
# copy_img51_1.extend(copy_img51_5)
len(copy_img71_1)
copy_img72_1 = [(np.copy([final_images[7]]), 2) for i in range(20)]
copy_img72_2 = [(np.copy([final_images[19]]), 2) for i in range(20)]
copy_img72_3 = [(np.copy([final_images[40]]), 2) for i in range(20)]
# copy_img52_4 = [(np.copy([final_images[63]]), 2) for i in range(20)]
# copy_img2_5 = [(np.copy(final_images[66]), 2) for i in range(20)]
# copy_img2_6 = [(np.copy(final_images[66]), 2) for i in range(20)]
copy_img72_1.extend(copy_img72_2)
copy_img72_1.extend(copy_img72_3)
# copy_img52_1.extend(copy_img52_4)
len(copy_img72_1)
copy_img73_1 = [(np.copy([final_images[15]]), 3) for i in range(20)]
copy_img73_2 = [(np.copy([final_images[18]]), 3) for i in range(20)]
copy_img73_3 = [(np.copy([final_images[28]]), 3) for i in range(20)]
copy_img73_4 = [(np.copy([final_images[57]]), 3) for i in range(20)]
# copy_img33_5 = [(np.copy([final_images[75]]), 3) for i in range(20)]
#copy_img3_6 = [(np.copy(final_images[66]), 3) for i in range(20)]
copy_img73_1.extend(copy_img73_2)
copy_img73_1.extend(copy_img73_3)
copy_img73_1.extend(copy_img73_4)
# copy_img33_1.extend(copy_img33_5)
len(copy_img73_1)
plt.imshow(new_img)
copy_img74_1 = [(np.copy([final_images[12]]), 4) for i in range(20)]
copy_img74_2 = [(np.copy([final_images[20]]), 4) for i in range(20)]
copy_img74_3 = [(np.copy([final_images[53]]), 4) for i in range(20)]
copy_img74_4 = [(np.copy([final_images[70]]), 4) for i in range(20)]
#copy_img4_5 = [(np.copy(final_images[60]), 4) for i in range(20)]
#copy_img4_6 = [(np.copy(final_images[66]), 4) for i in range(20)]
copy_img74_1.extend(copy_img74_2)
copy_img74_1.extend(copy_img74_3)
copy_img74_1.extend(copy_img74_4)
len(copy_img74_1)
copy_img75_1 = [(np.copy([final_images[13]]), 5) for i in range(20)]
copy_img75_2 = [(np.copy([final_images[34]]), 5) for i in range(20)]
copy_img75_3 = [(np.copy([final_images[60]]), 5) for i in range(20)]
copy_img75_4 = [(np.copy([final_images[73]]), 5) for i in range(20)]
# copy_img55_5 = [(np.copy([final_images[76]]), 5) for i in range(20)]
# copy_img25_6 = [(np.copy(final_images[66]), 5) for i in range(20)]
copy_img75_1.extend(copy_img75_2)
copy_img75_1.extend(copy_img75_3)
copy_img75_1.extend(copy_img75_4)
# copy_img55_1.extend(copy_img55_5)
len(copy_img75_1)
plt.imshow(final_images[68])
copy_img76_1 = [(np.copy([final_images[62]]), 6) for i in range(20)]
copy_img76_2 = [(np.copy([final_images[68]]), 6) for i in range(20)]
# copy_img66_3 = [(np.copy([final_images[54]]), 6) for i in range(20)]
#copy_img26_4 = [(np.copy([final_images[]]), 6) for i in range(20)]
#copy_img6_5 = [(np.copy(final_images[57]), 6) for i in range(20)]
#copy_img6_6 = [(np.copy(final_images[6]), 6) for i in range(20)]
copy_img76_1.extend(copy_img76_2)
# copy_img56_1.extend(copy_img56_3)
# copy_img36_1.extend(copy_img36_4)
len(copy_img76_1)
# copy_img26_1.extend(copy_img26_4)
plt.imshow(new_img)
plt.imshow(final_images[67])
copy_img67_1 = [(np.copy([final_images[13]]), 7) for i in range(20)]
copy_img67_2 = [(np.copy([final_images[50]]), 7) for i in range(20)]
# copy_img57_3 = [(np.copy([final_images[]]), 7) for i in range(20)]
# copy_img57_4 = [(np.copy([final_images[57]]), 7) for i in range(20)]
# copy_img57_5 = [(np.copy([final_images[71]]), 7) for i in range(20)]
#copy_img27_6 = [(np.copy([final_images[57]]), 7) for i in range(20)]
#copy_img27_7 = [(np.copy([final_images[69]]), 7) for i in range(20)]
copy_img67_1.extend(copy_img67_2)
# copy_img57_1.extend(copy_img57_3)
# copy_img57_1.extend(copy_img57_4)
# copy_img57_1.extend(copy_img57_5)
#copy_img57_1.extend(copy_img27_6)
#copy_img27_1.extend(copy_img27_7)
len(copy_img67_1)
copy_img78_1 = [(np.copy([final_images[27]]), 8) for i in range(20)]
copy_img78_2 = [(np.copy([final_images[41]]), 8) for i in range(20)]
copy_img78_3 = [(np.copy([final_images[52]]), 8) for i in range(20)]
copy_img78_4 = [(np.copy([final_images[67]]), 8) for i in range(20)]
# copy_img18_5 = [(np.copy([final_images[54]]), 8) for i in range(20)]
# copy_img8_6 = [(np.copy(final_images[66]), 8) for i in range(20)]
copy_img78_1.extend(copy_img78_2)
copy_img78_1.extend(copy_img78_3)
copy_img78_1.extend(copy_img78_4)
# copy_img28_1.extend(copy_img28_5)
len(copy_img78_1)
plt.imshow(new_img)
# copy_img1_1[0][0].shape
copy_img79_1 = [(np.copy([final_images[0]]), 9) for i in range(22)]
copy_img79_2 = [(np.copy([final_images[23]]), 9) for i in range(22)]
copy_img79_3 = [(np.copy([final_images[39]]), 9) for i in range(22)]
copy_img79_4 = [(np.copy([final_images[46]]), 9) for i in range(22)]
copy_img79_5 = [(np.copy([final_images[61]]), 9) for i in range(22)]
copy_img79_6 = [(np.copy([final_images[65]]), 9) for i in range(20)]
copy_img79_7 = [(np.copy([final_images[33]]), 9) for i in range(20)]
copy_img79_1.extend(copy_img79_2)
copy_img79_1.extend(copy_img79_3)
copy_img79_1.extend(copy_img79_4)
copy_img79_1.extend(copy_img79_5)
copy_img79_1.extend(copy_img79_6)
copy_img79_1.extend(copy_img79_7)
len(copy_img79_1)
plt.imshow(images_list[65])
#copy_img71_1.extend(copy_img72_1)
#copy_img71_1.extend(copy_img73_1)
#copy_img71_1.extend(copy_img74_1)
#copy_img71_1.extend(copy_img75_1)
#copy_img71_1.extend(copy_img76_1)
#copy_img71_1.extend(copy_img77_1)
copy_img71_1.extend(copy_img78_1)
copy_img71_1.extend(copy_img79_1)
len(copy_img71_1)
shuffle(copy_img71_1)
first_list = copy_img11_1
len(first_list)
second_list = copy_img21_1
len(second_list)
third_list = copy_img31_1
len(third_list)
fourth_list = copy_img41_1
len(fourth_list)
fifth_list = copy_img51_1
len(fifth_list)
sixth_list = copy_img61_1
len(sixth_list)
seventh_list = copy_img71_1
len(seventh_list)
# plt.imshow(second_list[0][0])
first_list.extend(second_list)
first_list.extend(third_list)
first_list.extend(fourth_list)
first_list.extend(fifth_list)
first_list.extend(sixth_list)
first_list.extend(seventh_list)
len(first_list)
shuffle(first_list)
#first_list = copy_img1_1
print(len(first_list))
len(second_list)
#type(first_list)
# helper.imshow(first_list[0][0])
first_list[0][1]<jupyter_output><empty_output><jupyter_text>Pickle the files<jupyter_code>import pickle
with open('final_list', 'wb') as f:
    pickle.dump(first_list, f)<jupyter_output><empty_output><jupyter_text>Transform numpy arrays to tensors<jupyter_code>transform = transforms.ToTensor()
tensors = []
for num in first_list:
    tensors.append((transform(num[0]).view(1,28,28).type(torch.FloatTensor), num[1]))
        
#    for i, b in num: #num represents the tuple itself
#        i = transform(i)
#        tensors.append((i,b))
#        transform(copy_img1_1[i][b])
type(tensors)
type(tensors[200])
(tensors[203][0]).dtype
len(tensors)
tensors[20][0].shape # 1 is the number of images, 1 is the layers i.e. RGB vs Gray and 28*28 is the size
# train_loader = (n for n in tensors)
# batch = next(iter(train_loader))
# images, labels = batch
# images, labels = batch

train_loader = torch.utils.data.DataLoader(tensors, batch_size=10, shuffle=True)
batch = next(iter(train_loader))
print('len:', len(batch))
images, labels = batch
print('Types:', type(images), type(labels))
print('shapes:', images.shape, labels.shape)

labels
import torchvision
grid = torchvision.utils.make_grid(images, nrow=10)

plt.figure(figsize=(15,15))
# plt.imshow(grid)
plt.imshow(np.transpose(grid, (1,2,0)))

print('labels:', labels)
import helper
# images, label = next(iter(train_loader))
helper.imshow(images[4,:]);<jupyter_output><empty_output><jupyter_text>Models We will be creating a CNN to train our model.<jupyter_code># Build our own network.
class ConvolutionalNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 6, 3, 1)
        self.conv2 = nn.Conv2d(6, 16, 3, 1)
        self.fc1 = nn.Linear(5*5*16, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84,10)

    def forward(self, X):
        X = F.relu(self.conv1(X))
        X = F.max_pool2d(X, 2, 2)
        X = F.relu(self.conv2(X))
        X = F.max_pool2d(X, 2, 2)
        X = X.view(-1, 5*5*16)
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.fc3(X)
        return F.log_softmax(X, dim=1)
torch.manual_seed(42)
model = ConvolutionalNetwork()
model
train_loader = torch.utils.data.DataLoader(tensors, batch_size=10, shuffle=True)
batch = next(iter(train_loader))
print('len:', len(batch))
images, labels = batch
print('Types:', type(images), type(labels))
print('shapes:', images.shape, labels.shape)

labels
preds = model(images) # Pass Batch
loss = F.cross_entropy(preds, labels) # Calculate Loss

loss.backward() # Calculate Gradients
optimizer.step() # Update Weights

print('loss1:', loss.item())
preds = model(images)
loss = F.cross_entropy(preds, labels)
print('loss2:', loss.item())
network = ConvolutionalNetwork()
optimizer = optim.Adam(network.parameters(), lr=0.01)
train_loader = torch.utils.data.DataLoader(
    tensors
    ,batch_size=100
    ,shuffle=True
)
def get_num_correct(preds, labels):
    return preds.argmax(dim=1).eq(labels).sum().item()
@torch.no_grad()
def get_all_preds(model, loader):
    all_preds = torch.tensor([])
    for batch in loader:
        images, labels = batch

        preds = model(images)
        all_preds = torch.cat(
            (all_preds, preds)
            ,dim=0
        )
    return all_preds
for epoch in range(50):

    total_loss = 0
    total_correct = 0

    for batch in train_loader: # Get Batch
        images, labels = batch 

        preds = network(images) # Pass Batch
        loss = F.cross_entropy(preds, labels) # Calculate Loss

        optimizer.zero_grad()
        loss.backward() # Calculate Gradients
        optimizer.step() # Update Weights

        total_loss += loss.item()
        total_correct += get_num_correct(preds, labels)

    print(
        "epoch", epoch, 
        "total_correct:", total_correct, 
        "loss:", total_loss
    )
images.shape
model
checkpoint = {'model': model,
          'state_dict': model.state_dict(),
          'optimizer' : optimizer.state_dict()}

torch.save(checkpoint, 'checkpoint1.pth')
torch.save(model, 'model1.pth')
torch.save(model.state_dict(), 'model1.pth')
#the_model = TheModelClass(*args, **kwargs)
model.load_state_dict(torch.load('model1.pth'))<jupyter_output><empty_output><jupyter_text>Making Predictions<jupyter_code>model
transform = transforms.ToTensor()
tensor_images = []
for images in final_images:
    tensors = torch.tensor(images, dtype=torch.float)
    tensor_images.append(tensors)
model(tensor_images[79].view(1,1,28,28)).argmax()
IMG_DIR = 'C:/Users/Billi/repos/sudoku/images/Square5'
filenames = os.listdir(IMG_DIR)
#filenames.sort()
final_images1 = []
for images in filenames:
    img_array = cv2.imread(os.path.join(IMG_DIR, images))
#    img_array = cv2.cvtColor(img_array, cv2.COLOR_BGR2GRAY)
#    resize_img = cv2.resize(img_array, (28,28))
    final_images1.append(img_array)
#pred_tensor = []
#for num in final_images1:
#pred_tensor.append((transform(num[0]).view(1,28,28).type(torch.FloatTensor), num[1]))
for i in range(len(final_images1)):
    if final_images1[i].max() != 255:
        np.full_like(final_images1[i], 0)
plt.imshow(final_images1[4])
final_images1[4].mean()
transform = transforms.ToTensor()
tensor_images1 = []
for images in final_images1:
    tensors1 = torch.tensor(images, dtype=torch.float)
    tensor_images1.append(tensors1)
tensor_images1[5].shape
def convert(tensor): # always be passing in an array
    grid = []
    for i in range(len(tensor_images1)):
        if tensor[i].mean().item() <= 22:
            grid.append(".")
        else:
            grid.append(str(network(tensor[i]).argmax().item()))
    return grid
convert(tensor_images1)
len(tensors)
optimizer = optim.Adam(network.parameters(), lr=0.01)

total_loss = 0
total_correct = 0

for batch in next(iter(train_loader)): # Get Batch
    images, labels = batch 

    preds = network(images) # Pass Batch
    loss = F.cross_entropy(preds, labels) # Calculate Loss

    optimizer.zero_grad()
    loss.backward() # Calculate Gradients
    optimizer.step() # Update Weights

    total_loss += loss.item()
#    total_correct += get_num_correct(preds, labels)
    
print(
    "epoch:", 0, 
#    "total_correct:", total_correct, 
    "loss:", total_loss
)

criterion = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.003)

epochs = 25
for e in range(epochs):
    running_loss = 0
    for images, labels in next(iter(train_loader)):
        # Flatten MNIST images into a 784 long vector
        images = images.view(1,28,28).argmax()
    
        # TODO: Training pass
        optimizer.zero_grad()
        
        output = model(images)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        
        running_loss += loss.item()
    else:
        print(f"Training loss: {running_loss/len(train_loader)}")
def convert(tensor): # always be passing in an array
    grid = []
    for i in range(len(tensor)):
        if tensor[i].mean().item() <= 22:
            grid.append(".")
        else:
            grid.append(str(model(tensor[i].view(1,1,28,28)).argmax().item()))
    return grid
class ConvolutionalNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 6, 3, 1)
        self.conv2 = nn.Conv2d(6, 16, 3, 1)
        self.fc1 = nn.Linear(5*5*16, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84,10)

    def forward(self, X):
        X = F.relu(self.conv1(X))
        X = F.max_pool2d(X, 2, 2)
        X = F.relu(self.conv2(X))
        X = F.max_pool2d(X, 2, 2)
        X = X.view(-1, 5*5*16)
        X = F.relu(self.fc1(X))
        X = F.relu(self.fc2(X))
        X = self.fc3(X)
        return F.log_softmax(X, dim=1)
torch.manual_seed(42)
model = ConvolutionalNetwork()
model
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
import time
start_time = time.time()

epochs = 5
train_losses = []
test_losses = []
train_correct = []
test_correct = []

for i in range(epochs):
    trn_corr = 0
    tst_corr = 0
    
    # Run the training batches
    for b, (X_train, y_train) in enumerate(train_loader):
        b+=1
        
        # Apply the model
        y_pred = model(X_train)  # we don't flatten X-train here
        loss = criterion(y_pred, y_train)
 
        # Tally the number of correct predictions
        predicted = torch.max(y_pred.data, 1)[1]
        batch_corr = (predicted == y_train).sum()
        trn_corr += batch_corr
        
        # Update parameters
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        # Print interim results
#        if b%600 == 0:
#            print(f'epoch: {i:2}  batch: {b:4} [{10*b:6}/60000]  loss: {loss.item():10.8f}  \
#accuracy: {trn_corr.item()*100/(10*b):7.3f}%')

for i in range(len(final_images)):
    if final_images[i].max() != 255:
        np.full_like(final_images[i], 0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15352963180676535, 'fraction_numerical': 0.06722599262544755, 'mean_word_length': 4.8646192416170475, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '370373', 'license': 'MIT', 'n_tokens_mistral': 8038, 'n_tokens_neox': 7350, 'n_words': 1403, 'path': 'Notebooks/CNN Model/PreprocessingPipeline.ipynb', 'repo_name': 'supai-red/omega2020-ds', 'size': '18713'}