<jupyter_start><jupyter_text><jupyter_code>import torch
from torchvision import models, transforms, utils
from torch.utils.data import Dataset, DataLoader
from torch.utils.tensorboard import SummaryWriter
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
import pickle
import matplotlib.pyplot as plt
%matplotlib inline
import math
from sklearn.metrics import confusion_matrix
import warnings

warnings.filterwarnings("ignore")
# from google.colab import drive
# drive.mount('/content/drive/')
# X = np.load('drive/My Drive/Copy of xtrain.npy')
# y = np.load('drive/My Drive/Copy of ytrain.npy')
X_train = pickle.load(open(f'X_train298.pkl', 'rb'))
y_train = pickle.load(open(f'y_train298.pkl', 'rb'))
y_test = pickle.load(open(f'y_test298.pkl', 'rb'))
X_test = pickle.load(open(f'X_test298.pkl', 'rb'))
X_test.shape
# X = torch.from_numpy(X)
# X = X.permute(0,3,1,2)
# y = torch.from_numpy(y)
# X.shape, y.shape
# pickle.dump(X, open("augX.pkl", 'wb'), protocol=4)
# pickle.dump(resized_imageset, open(f'resized_imageset1.pkl', 'wb'))
# X = pickle.load(open(f'resized_imageset1.pkl', 'rb'))
# X_train = pickle.load(open(f'xtrain298.pkl', 'rb'))
# y_train = pickle.load(open(f'ytrain298.pkl', 'rb'))
# X_test = pickle.load(open(f'xtest298.pkl', 'rb'))
# y_test = pickle.load(open(f'ytest298.pkl', 'rb'))
# def splitTrainTest(X,y):
#   shuffled_indices = torch.randperm(X.shape[0])
#   ul = math.floor(0.8*X.shape[0])
#   train_indices = shuffled_indices[:ul]
#   test_indices = shuffled_indices[ul:]
#   # train_indices.shape[0] + test_indices.shape[0]
#   X_train = X[train_indices]
#   y_train = y[train_indices]  
#   X_test = X[test_indices]
#   y_test = y[test_indices]
#   print('y_train -> [0]:{} [1]:{}'.format((y_train == 0).sum().item(), (y_train == 1).sum().item()))
#   print('y_test -> [0]:{} [1]:{}'.format((y_test == 0).sum().item(), (y_test == 1).sum().item()))
#   return X_train, y_train, X_test, y_test
# X_train, y_train, X_test, y_test = splitTrainTest(X, y)
# pickle.dump(X_train, open("X_train298.pkl", 'wb'), protocol=4)
# pickle.dump(X_test, open("X_test298.pkl", 'wb'), protocol=4)
# pickle.dump(y_train, open("y_train298.pkl", 'wb'), protocol=4)
# pickle.dump(y_test, open("y_test298.pkl", 'wb'), protocol=4)<jupyter_output><empty_output><jupyter_text>Normalizing<jupyter_code>def getNormalized(X,ns):
  flattened_channels = X.reshape(3,-1)
  channel_mean = flattened_channels.mean(dim = 1)
  channel_stddev = flattened_channels.std(dim = 1)
  preprocess2 = transforms.Compose([
                      transforms.Normalize(channel_mean, channel_stddev)
  ])


  temptwo = torch.tensor([])
  for i in range(X.shape[0]):
    a = preprocess2(X[i])
    temptwo = torch.cat([temptwo, a.reshape(1,3,ns,ns)])
  
  return temptwo<jupyter_output><empty_output><jupyter_text>Resizing<jupyter_code>def imageSetResize(newSize,X):
  preprocess1 = transforms.Compose([
                        transforms.ToPILImage(),
                        transforms.Resize(newSize),
                        transforms.ToTensor()])
  
  temp = torch.tensor([])
  for i in range(X.shape[0]):
    a = preprocess1(X[i])
    temp = torch.cat([temp, a.reshape(1,3,newSize,newSize)])

  return temp 
imsize = 256
X_train = imageSetResize(imsize, X_train.float())
X_train = getNormalized(X_train.float(),imsize)
X_test = imageSetResize(imsize, X_test.float())
X_test = getNormalized(X_test.float(),imsize)
X_train.shape<jupyter_output><empty_output><jupyter_text>Checking Distribution<jupyter_code>(y_test == 1.).sum(), (y_test == 0.).sum(), (y_train == 0.).sum(), (y_train == 1.).sum()<jupyter_output><empty_output><jupyter_text>Alexnet<jupyter_code>anet = models.alexnet(pretrained = True)
def giveAlexnetRepresentations(X):
  anet_representations = torch.tensor([])

  for i in range(X.shape[0]):
    anet_representations = torch.cat([anet_representations, anet(X[i].unsqueeze(0))])

  return anet_representations 
model1 = nn.Sequential(
    nn.Linear(1000, 300),
    nn.Dropout(p=0.3),
    nn.ReLU(),
    nn.Linear(300,100),
    nn.Dropout(p=0.6),
    nn.ReLU(),
    nn.Linear(100,2),
    nn.Dropout(p=0.5),
    nn.ReLU(),
    nn.Softmax(dim=1)
)<jupyter_output><empty_output><jupyter_text>Operations Shuffling and Batching<jupyter_code>def labelize(p):
  labelized_preds = []
  for i in p:
    l = 0. if i[0]>i[1] else 1.
    labelized_preds.append(l)

  return torch.tensor(labelized_preds)
def shuffle_and_batch(X,y,num,bs):
  shuffled_indices = torch.randperm(X.shape[0])
  newX = X[shuffled_indices]
  newY = y[shuffled_indices]

  X_batches = []
  y_batches = []
  for i in range(num):
    X_batches.append(X[i*bs:(i+1)*bs])
    y_batches.append(y_train[i*bs:(i+1)*bs])

  return X_batches, y_batches
X_train_representations = giveAlexnetRepresentations(X_train)
X_test_representations = giveAlexnetRepresentations(X_test.float())
X_train_representations.shape, X_test_representations.shape<jupyter_output><empty_output><jupyter_text>Training and Testing<jupyter_code># device = torch.device('cpu')
# model1 = model1.to(device)
# tb = SummaryWriter()
prev_testacc = -float('inf')
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(lr = 1e-3, params = model1.parameters())
model_path = './salexnet.pth'
prev_testacc = -float('inf')


for epoch in range(10):
    X_batches, y_batches = shuffle_and_batch(X_train_representations, y_train, 4, 67)
  # X_batches = X_batches.to(device)
  # y_batches = y_batches.to(device)
    print('Epoch {}'.format(epoch))
    for i in range(1):#len(X_batches)):
        pred = model1(X_batches[i])
        loss = criterion(pred,y_batches[i].long())
        optimizer.zero_grad()
        loss.backward(retain_graph = True)
        optimizer.step()


  # Checking model on training set
    train_preds = model1(X_train_representations)
    train_loss = criterion(train_preds, y_train.long())
    train_preds = labelize(train_preds)
    train_prediction_comparisons = (y_train == train_preds)
    train_accuracy = float(train_prediction_comparisons.sum())/float(y_train.shape[0])
    print('Training Loss:{}\tTraining Accuracy:{}'.format(train_loss.item(), train_accuracy), end='  ')
  
  # Checking model on testing set
    test_preds = model1(X_test_representations)
    test_loss = criterion(test_preds, y_test.long())
    test_preds = labelize(test_preds)
    test_prediction_comparisons = (y_test == test_preds)
    test_accuracy = float(test_prediction_comparisons.sum())/float(y_test.shape[0])
    print('Testing Loss:{}\tTesting Accuracy:{}'.format(test_loss.item(), test_accuracy))
    if test_accuracy < prev_testacc and prev_testacc>0.7:
        break
#     state_dict = model1.state_dict()
    torch.save(model1, model_path)
    prev_testacc = test_accuracy


#     tb.add_scalar('TrainLoss',train_loss, epoch)
#     tb.add_scalar('TestLoss',test_loss, epoch)
#     tb.add_scalar('TrainAccuracy', train_accuracy, epoch)
#     tb.add_scalar('TestAccuracy', test_accuracy, epoch)

confusion_matrix(y_train, train_preds)
confusion_matrix(y_test, test_preds)<jupyter_output><empty_output><jupyter_text>AugSet<jupyter_code># Xaug = pickle.load(open(f'X5040_Alexnet.pkl', 'rb'))
# yaug = pickle.load(open(f'y5040.pkl', 'rb'))
# aug_preds = model1(Xaug)
# aug_loss = criterion(aug_preds, yaug.long())
# aug_preds = labelize(aug_preds)
# aug_prediction_comparisons = (yaug == aug_preds)
# aug_accuracy = float(aug_prediction_comparisons.sum())/float(yaug.shape[0])
# print('AugAccuracy:{}'.format(aug_accuracy), end='  ') 
# confusion_matrix(yaug, aug_preds)<jupyter_output><empty_output><jupyter_text>Raw<jupyter_code>Xraw = pickle.load(open(f'XAlexnet2669_raw_256.pkl', 'rb'))
yraw = pickle.load(open(f'yAlexnet2669_raw_256.pkl', 'rb'))
a1 = torch.load(model_path)
# raw_preds = model11(Xraw)
# # raw_loss = criterion(raw_preds, labelToOneHot(yraw))
# raw_preds = labelize(raw_preds)
# raw_prediction_comparisons = (yraw == raw_preds)
# raw_accuracy = float(raw_prediction_comparisons.sum())/float(yraw.shape[0])
# print('RawAccuracy:{}'.format(raw_accuracy), end='  ')  
# confusion_matrix(yraw, raw_preds)
# from sklearn.metrics import f1_score
# f1_score(yraw, raw_preds, average = 'weighted')
test_preds = a1(X_test_representations)
test_loss = criterion(test_preds, y_test.long())
#     test_loss = criterion(test_preds,labelToOneHot(y_test))
test_preds = labelize(test_preds)
test_prediction_comparisons = (y_test == test_preds)
test_accuracy = float(test_prediction_comparisons.sum())/float(y_test.shape[0])
print('TestLoss:{} TestAccuracy:{}'.format(test_loss.item(), test_accuracy))<jupyter_output>TestLoss:0.6931471228599548 TestAccuracy:0.5<jupyter_text>Densenet<jupyter_code># dnet = models.densenet121(pretrained = True)
# normalized_imageset = getNormalized(X.float())
# resized_imageset = imageSetResize(256, normalized_imageset)
# def getD121Representations(X):
#   global dnet
#   device = torch.device('cuda:0')
#   dnet_representations = torch.tensor([]).float().to(device)
#   dnet = dnet.to(device)

#   X = X.float().to(device)
#   for i in range(X.shape[0]):
#     dnet_representations = torch.cat([dnet_representations, dnet(X[i].unsqueeze(0))])

#   device = torch.device('cpu')
#   dnet_representations = dnet_representations.to(device)
#   X = X.to(device)
#   dnet = dnet.to(device)
#   return dnet_representations 
# X_train_representations = getD121Representations(X_train)
# X_test_representations = getD121Representations(X_test)
# device = torch.device('cpu')
# model1 = model1.to(device)



# criterion = nn.CrossEntropyLoss()
# optimizer = optim.Adam(lr = 1e-4, params = model1.parameters())
# optimizer.zero_grad()
# for epoch in range(10):
#   X_batches, y_batches = shuffle_and_batch(X_train_representations, y_train, 4, 67)
#   # X_batches = X_batches.to(device)
#   # y_batches = y_batches.to(device)
#   print('Epoch {}'.format(epoch))
#   for i in range(1):#len(X_batches)):
#     pred = model1(X_batches[i])
#     loss = criterion(pred,y_batches[i].long())
#     # print('Training Loss: {}'.format(loss))
#     loss.backward(retain_graph = True)
#     optimizer.step()


#   # Checking model on training set
#   train_preds = model1(X_train_representations)
#   train_loss = criterion(train_preds, y_train.long())
#   train_preds = labelize(train_preds)
#   train_prediction_comparisons = (y_train == train_preds)
#   train_accuracy = float(train_prediction_comparisons.sum())/float(y_train.shape[0])
#   print('Training Loss:{}\tTraining Accuracy:{}'.format(train_loss.item(), train_accuracy), end='  ')
  
#   # Checking model on testing set
#   test_preds = model1(X_test_representations)
#   test_loss = criterion(test_preds, y_test.long())
#   test_preds = labelize(test_preds)
#   test_prediction_comparisons = (y_test == test_preds)
#   test_accuracy = float(test_prediction_comparisons.sum())/float(y_test.shape[0])
#   print('Testing Loss:{}\tTesting Accuracy:{}'.format(test_loss.item(), test_accuracy))

# torch.cuda.empty_cache()
# if torch.cuda.is_available():
  # device = torch.device('cuda:0')
  # print('{} GPU(s)'.format(torch.cuda.device_count()))
# else:
  # device = torch.device('cpu')
  # print('CPU')
# !ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi
# !pip install gputil
# !pip install psutil
# !pip install humanize
# import psutil
# import humanize
# import os
# import GPUtil as GPU
# GPUs = GPU.getGPUs()
# # XXX: only one GPU on Colab and isn’t guaranteed
# gpu = GPUs[0]
# def printm():
#  process = psutil.Process(os.getpid())
#  print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
#  print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))
# printm()
# def printm():
#  process = psutil.Process(os.getpid())
#  print("Gen RAM Free: " + humanize.naturalsize( psutil.virtual_memory().available ), " | Proc size: " + humanize.naturalsize( process.memory_info().rss))
#  print("GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))

# torch.cuda.empty_cache()
# printm()
16280/1024
# torch.cuda.is_available()
# device = torch.device('cuda:0')
# device, torch.cuda.device_count()
# xxxx = torch.from_numpy(X)
# XXXX = xxxx[:4]
# XXXX = XXXX.permute(0,3,1,2)

# flattened_channels = XXXX.reshape(3,-1)
# channel_mean = flattened_channels.mean(dim = 1)
# channel_stddev = flattened_channels.std(dim = 1)
# pppp = transforms.Compose([
                          # transforms.Normalize(channel_mean, channel_stddev)
# ])


############################################################
# import torch.multiprocessing as mp
# mp.cpu_count()
###########################################################
# class PCBdataset(Dataset):
#   def __init__(self, numpy_file, label_file, transform):
#     self.image_file = numpy_file
#     self.label_file = label_file
#     self.transform = transform
    
    
#   def __len__(self):
#     return self.image_file.shape[0]

#   def __getitem__(self, idx):

#     image = self.image_file[idx]
#     if(self.transform):
#       image = self.transform(self.image_file[idx])

#     di = {'image':image, 'label':self.label_file[idx].item()}
#     return di


# .permute(1,2,0)
# import torchvision.transforms.functional as F
# t = F.to_pil_image(a[0]['image'])
# a = PCBdata[0]['image']
# plt.imshow(a)
# a = torch.tensor([1.,2.,3.,4.])

# b = a[0]
# c = a[1]
# d = a[2]
# e = a[3]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16610343814281517, 'fraction_numerical': 0.020570085218924478, 'mean_word_length': 5.247361174850849, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '553607', 'license': 'MIT', 'n_tokens_mistral': 5411, 'n_tokens_neox': 5175, 'n_words': 801, 'path': 'training/salexnet.ipynb', 'repo_name': 'shouvikcirca/PCB_Defect_Detection', 'size': '13612'}