<jupyter_start><jupyter_text><jupyter_code>from torch.utils.data import Dataset, DataLoader
import glob
import cv2
import torch
import pickle
from google.colab import drive
drive.mount('/content/gdrive')
# Tensor parameters
coeff=1
size = (64,64)
tensor_type='RGB'
#parent_dir = '/content/gdrive/MyDrive/CS_EE_577/Dataset/'
def get_reshape_dims(size,tensor_type,coeff):
   mapping = {}
   coeff_mapping = {}
   mapping['RGB'] = 3
   mapping['RGBZ'] = 6
   mapping['Z'] = 3 
   coeff_mapping[1] = 17
   coeff_mapping[2] = 81
   return (mapping[tensor_type]*coeff_mapping[coeff],int(size[0]/4),int(size[0]/4))

reshape_dims = get_reshape_dims(size,tensor_type,coeff)
class CustomDataset(Dataset):
  def __init__(self, transforms=None):
    self.dataset_path = f'/content/gdrive/MyDrive/Colab Notebooks/CS_EE_577/Dataset/Tensors/Coeff_Order{coeff}/{tensor_type}_{size[0]}_{size[0]}/'  
    directory_list = glob.glob(self.dataset_path + "*")     ### This list should contain paths for both melanoma and nonmelanoma folders(containing tensors)
    self.data = []                        ### List containing list of tensor_img and class_name [[tensor_img1, class_name1],[tensor_img2, class_name2]....]
    self.transforms = transforms
    for single_class in directory_list: 
      class_name = single_class.split("/")[-1]
      for tensor_img in glob.glob(single_class + "/*.pickle"):  ### Change the format of .png based on what the tensors are ? 
        with open(tensor_img, 'rb') as f:
          res = pickle.load(f)
        res = torch.reshape(res, reshape_dims)
        self.data.append([res, class_name])
        
    self.class_map = {"Melanoma" : 1, "Non_Melanoma": 0} 

  def __len__(self):
    return len(self.data)    
  
  def __getitem__(self, idx):
    tensor_image, class_name = self.data[idx]
    class_id = self.class_map[class_name]
    #img_tensor = torch.from_numpy(img)   ### Exclude this if they are already a torch tensor
    class_id = torch.tensor([class_id])
    if self.transforms is not None:
      tensor_image = self.transforms(tensor_image)
    return tensor_image, class_id
import torch
from torch.nn import Module
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.functional import cross_entropy, relu
from torch.nn.parameter import Parameter
from torch.optim import SGD
from torch.utils.data import DataLoader, TensorDataset, Dataset
import torch.utils.data as data_utils

train_on_gpu = torch.cuda.is_available()

class melanomaNetwork(Module):
	def __init__(self, classes = 2):
		super(melanomaNetwork, self).__init__()
		self.classes = classes

		#fitting
		self.fitting_input_channels = reshape_dims[0]
		self.fitting_kernels = 32

		#structure1 wb1
		self.s1_kernels = 64
		self.s1wb1_conv1 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb1_conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb1_conv3 = nn.Conv2d(32, 64, kernel_size=1, stride=1, padding=0)
		self.s1wb1_bn1 = nn.BatchNorm2d(64)
		self.s1wb1_bn2 = nn.BatchNorm2d(64)
		self.s1wb1_bn3 = nn.BatchNorm2d(64)


		#structure1 wb2
		self.s1wb2_conv1 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb2_conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb2_bn1 = nn.BatchNorm2d(64)
		self.s1wb2_bn2 = nn.BatchNorm2d(64)


		#structure1 wb3
		self.s1wb3_conv1 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb3_conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) 
		self.s1wb3_bn1 = nn.BatchNorm2d(64)
		self.s1wb3_bn2 = nn.BatchNorm2d(64)


		#structure1 wb4 
		self.s1wb4_conv1 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb4_conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1)
		self.s1wb4_bn1 = nn.BatchNorm2d(64)
		self.s1wb4_bn2 = nn.BatchNorm2d(64)


		#structure2 wb1
		self.s2_kernels = 128
		self.s2wb1_conv1 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb1_conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb1_conv3 = nn.Conv2d(64, 128, kernel_size=1, stride=1, padding=0)
		self.s2wb1_bn1 =  nn.BatchNorm2d(128)
		self.s2wb1_bn2 =  nn.BatchNorm2d(128)
		self.s2wb1_bn3 = nn.BatchNorm2d(128)

		#structure2 wb2 
		self.s2wb2_conv1 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb2_conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb2_bn1 = nn.BatchNorm2d(128)
		self.s2wb2_bn2 = nn.BatchNorm2d(128)

		#structure2 wb3
		self.s2wb3_conv1 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb3_conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb3_bn1 = nn.BatchNorm2d(128)
		self.s2wb3_bn2 = nn.BatchNorm2d(128)

		#structure2 wb4
		self.s2wb4_conv1 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb4_conv2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)
		self.s2wb4_bn1 = nn.BatchNorm2d(128)
		self.s2wb4_bn2 = nn.BatchNorm2d(128)

		self.stride = 1
		self.padding = 0
		self.kernel_size = 3 
		self.residue_kernel_size = 1
		
		self.bn1 = nn.BatchNorm2d(self.fitting_input_channels)
		self.conv1 = nn.Conv2d(self.fitting_input_channels, self.fitting_kernels, kernel_size=3, stride=1, padding=1)
		self.bn2 = nn.BatchNorm2d(self.fitting_kernels)

		#discrimination
		#self.averagepooling = nn.AvgPool2d(kernel_size=3, stride=1)
		self.adaptive_avg_pooling = nn.AdaptiveAvgPool2d(2)
		self.fc = nn.Linear(128*2*2, self.classes)

	def forward(self, x):

		#fitting
		x = F.relu(self.bn2(self.conv1(self.bn1(x))))
		#print("Size of X after fitting : ", x.size())

		#EXPANDING
		#STRUCTURE 1

		#wb1
		x = x 
		residue1 = x
		x = F.relu(self.s1wb1_bn1(self.s1wb1_conv1(x)))
		x = self.s1wb1_bn2(self.s1wb1_conv2(x))
		residue1 = self.s1wb1_bn3(self.s1wb1_conv3(residue1))
		x = x + residue1
		x = F.relu(x)

		#wb2
		x = F.relu(self.s1wb2_bn1(self.s1wb2_conv1(x)))
		x = F.relu(self.s1wb2_bn2(self.s1wb2_conv2(x)))

		#wb3
		x = F.relu(self.s1wb3_bn1(self.s1wb3_conv1(x)))
		x = F.relu(self.s1wb3_bn2(self.s1wb3_conv2(x)))

		#wb4
		x = F.relu(self.s1wb4_bn1(self.s1wb4_conv1(x)))
		x = F.relu(self.s1wb4_bn2(self.s1wb4_conv2(x)))

		#print("SIZE AFTER STRUCTURE1, ", x.size())

		#STRUCTURE2
		#wb1
		x = x 
		residue2 = x
		x = F.relu(self.s2wb1_bn1(self.s2wb1_conv1(x)))
		x = self.s2wb1_bn2(self.s2wb1_conv2(x))
		residue2 = self.s2wb1_bn3(self.s2wb1_conv3(residue1))
		x = x + residue2
		x = F.relu(x)

		#wb2
		x = F.relu(self.s2wb2_bn1(self.s2wb2_conv1(x)))
		x = F.relu(self.s2wb2_bn2(self.s2wb2_conv2(x)))

		#wb3
		x = F.relu(self.s2wb3_bn1(self.s2wb3_conv1(x)))
		x = F.relu(self.s2wb3_bn2(self.s2wb3_conv2(x)))

		#wb4
		x = F.relu(self.s2wb4_bn1(self.s2wb4_conv1(x)))
		x = F.relu(self.s2wb4_bn2(self.s2wb4_conv2(x)))

		#print("SIZE OF X AFTER STRUCTURE 2", x.size())
		
		#discrimination
		x = self.adaptive_avg_pooling(x)
		#print("SIZE OF X AFTER POOLING", x.size())

		x = x.view(x.size(0), -1)
		x = self.fc(x)
		
		x = F.softmax(x,dim=1)
		
		return x

from __future__ import print_function, division
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
import math
from typing import List
import matplotlib.pyplot as plt
import torch
from torch.distributions import Uniform
from torch.nn import Module
from torch.nn.functional import cross_entropy, relu
from torch.nn.parameter import Parameter
from torch.optim import Adam, SGD
from torch.utils.data import DataLoader, TensorDataset
import torch.utils.data as data_utils
from torch.autograd import Variable
from sklearn.metrics import balanced_accuracy_score


import math
from typing import List
import matplotlib.pyplot as plt
import torch
from torch.distributions import Uniform
from torch.nn import Module
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.functional import cross_entropy, relu
from torch.nn.parameter import Parameter
from torch.optim import Adam, SGD
from torch.utils.data import DataLoader, TensorDataset
import torch.utils.data as data_utils
from torch.utils.data import DataLoader, TensorDataset, Dataset
from torch.utils.data.sampler import SubsetRandomSampler
from torchvision import transforms
#from melanoma_dataloader import CustomDataset
import torch.utils.data as data
#from model_melanoma import Network
import glob
import cv2
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

def train(train_loader: DataLoader, val_loader:DataLoader, model: Module, optimizer : SGD, loss_criterion, total_epochs):
    accuracy = 0.0                  #can be used later to keep training loop with a specific accuracy
    train_loss_each_epoch = []      #contains average training loss in each epoch 
    val_loss_each_epoch = []        #contains average validation loss in each epoch 
    train_accuracy_each_epoch = []  #contains average training accuracy in each epoch
    val_accuracy_each_epoch = []    #contains average validation accuracy in each epoch
    train_bac_each_epoch = []       # contains bac after each epoch
    val_bac_each_epoch = []

    for epoch in range(total_epochs):

        print(f"Running epoch ", epoch, " of ", total_epochs)

        training_loss = float(0.0)
        
        correct_train_preds = 0
        '''
        Input: (N, C)(N,C) where C = number of classes,
        Target: If containing class indices, shape (N)(N) where each value is 0 \leq \text{targets}[i] \leq C-10≤targets[i]≤C−1,
        
        '''
        
        actual = []
        predicted = []

        for x, y in train_loader:
            if train_on_gpu:
              x, y = x.cuda(), y.cuda()
            x = model(x)
            y = torch.tensor(y, dtype=torch.float)
            x = torch.tensor(x, dtype=torch.float)

            x = Variable(x, requires_grad=True)
            y = Variable(y, requires_grad=True)
            y = y.squeeze()
            y = y.type(torch.float) #
            x = x.type(torch.float)

            # print(f"y shape {y.shape}")
            # print(f"x shape {x.shape}")

            loss = loss_criterion(x, y.long())
            # y = y.type(torch.long) #
            # x = x.type(torch.long)

            

            # #y = y.float()
            # y = y.to(torch.float32)
            # print("shapes")
            # print(f"y shape {y.shape}")
            # print(f"x shape {x.shape}")
            # print(y)
            # print(x)
            # print(type(x))
            # print(type(y))
            # loss = loss_criterion(x, y.long())
            # loss = loss_criterion(x, y.long())
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            x_pred = torch.argmax(x, axis=1)
            training_loss = training_loss + loss.item()
            # x_pred = torch.argmax(x, axis=1)
            correct_train_preds = correct_train_preds + torch.eq(x_pred, y).sum()

            actual.extend(y.detach().cpu().numpy())
            predicted.extend(x_pred.detach().cpu().numpy())
        
        
        train_accuracy = float(correct_train_preds/len(train_loader))
        
        # print("test")
        # print(len(actual))
        # print(len(predicted))
        # print(actual)
        # print(predicted)

        bac_score = balanced_accuracy_score(actual,predicted)
        
        train_loss = training_loss

        train_loss_each_epoch.append(train_loss)
        train_accuracy_each_epoch.append(train_accuracy)
        train_bac_each_epoch.append(bac_score)

        print(f"Training loss : {train_loss}, training accuracy : {train_accuracy}, training BAC: {bac_score} after epoch : {epoch}")

        print("EPOCHHH DONE NEXT UP ")
        validation_loss = float(0.0)
        correct_val_preds = 0 

        # TODO: 
        # Implement a stratified split to maintain min melanoma # in dataset
        # implement balanced accuracy 
        # cross validation - if we have time
        val_predict = []
        val_actual = []

        for x, y in val_loader:
            if train_on_gpu:
              x, y = x.cuda(), y.cuda()
            x = model(x)
            y = torch.tensor(y, dtype=torch.float)
            x = torch.tensor(x, dtype=torch.float)

            x = Variable(x, requires_grad=True)
            y = Variable(y, requires_grad=True)
            y = y.squeeze()
            y = y.type(torch.float) #
            x = x.type(torch.float)
            loss = loss_criterion(x, y.long())

            validation_loss += loss.item()  
            x_pred = torch.argmax(x, axis=1)
            correct_val_preds = correct_val_preds + torch.eq(x_pred, y).sum()

            val_actual.extend(y.detach().cpu().numpy())
            val_predict.extend(x_pred.detach().cpu().numpy())

        print("Val Actual: ")
        print(val_actual)
        print("Val Predict: ")
        print(val_predict)
        

        val_accuracy = float(correct_val_preds/len(val_loader))
        val_loss = float(validation_loss/len(val_loader))
        bac_score = balanced_accuracy_score(val_actual,val_predict)

        val_loss_each_epoch.append(val_loss)
        val_accuracy_each_epoch.append(val_accuracy)
        val_bac_each_epoch.append(bac_score)

        print(f"Validation loss : {val_loss}, validation accuracy : {val_accuracy}, bac score : {bac_score}  after epoch : {epoch}")
      
    return train_loss_each_epoch, train_accuracy_each_epoch, val_loss_each_epoch, val_accuracy_each_epoch, bac_score



#from model import melanomaNetwork, baseWorkBlock, firstWorkBlock
from sklearn.model_selection import KFold
def main():
	data_transforms = transforms.Compose([transforms.RandomRotation(degrees=(-90,90))])
	dataset = CustomDataset(transforms=data_transforms)
	dataset_x = [i for i in range(len(dataset))]
	dataset_y = [i[1] for i in dataset]
	# dataset_train, dataset_valid = data.random_split(dataset, (175,32))
	# targets = dataset.targets

	skf = KFold(n_splits=10, shuffle=True)
	skf.get_n_splits(dataset_x, dataset_y)
 
	total_bac_output = 0
	count = 0
 
	for train_index, test_index in skf.split(dataset_x, dataset_y):
		# returns indices in the dataset that corresponds to train and test set
		print(f"Train Index: {train_index}, size: {len(train_index)}")
		print(f"Test Index: {test_index}, size: {len(test_index)}")
		# X_train, X_test = dataset_x[train_index], dataset_x[test_index]
		# y_train, y_test = dataset_y[train_index], dataset_y[test_index]

		# X_train, X_test, y_train, y_test = train_test_split(dataset_x, dataset_y, test_size=0.2, shuffle=True, stratify=dataset_y)
		X_train = train_index
		X_test = test_index

		# generating a sampler object ... might not need
		# X_train_sampler = SubsetRandomSampler(X_train)
		# X_test_sampler = SubsetRandomSampler(X_test)
		# print(X_train)
		
		train_loader = 	DataLoader(dataset=dataset, batch_size=80, sampler=X_train)
		val_loader = DataLoader(dataset=dataset, batch_size=80, sampler=X_test)
		print("Length of the dataset:", dataset.__len__())	
		train_batch = 0
		img, label = next(iter(train_loader))
		print("IMAGE SIZE :",img.size())
		print("LABEL SIZE : ",label.size())
		network = melanomaNetwork()
		if train_on_gpu:
			network.cuda()
		print("NETWORK LOADED")
		
		optimizer = SGD(network.parameters(), lr=0.001,nesterov=True,momentum=0.9,weight_decay=0.01)
		#optimizer = Adam(network.parameters(), lr=0.0001)
		

		print("MODEL TRAINING STARTS")
		
		# #def train(train_loader: DataLoader, model: Module, optimizer : SGD, loss_criterion, total_epochs):
		loss = nn.CrossEntropyLoss()
		# loss = nn.BCELoss()
		#train_loss_each_epoch, train_accuracy_each_epoch, val_loss_each_epoch, val_accuracy_each_epoch = train(train_loader, network, optimizer, loss, 30)
		
		train_loss_each_epoch, train_accuracy_each_epoch, val_loss_each_epoch, val_accuracy_each_epoch, bac_output = train(train_loader, val_loader, network, optimizer, loss, 7)

		print("MODEL TRAINING IS FINISHED!")

		total_bac_output += bac_output
		count += 1

		#plt.plot(train_loss_each_epoch)
		#plt.show()
		# plt.plot(train_loss_each_epoch)
		# plt.show()
		# fig, (ax1, ax2, ax3, ax4) = plt.subplots(4,1,constrained_layout=True, figsize=(20, 10))
		# fig.set_figheight(15)
		# fig.set_figwidth(15)

		# ax1.plot(train_loss_each_epoch)
		# ax1.set_xlabel('Epochs')
		# ax1.set_ylabel('Training loss')
		# ax1.set_title('Training loss vs epochs')


	# ax2.plot(train_accuracy_each_epoch)
	# ax2.set_xlabel('Epochs')
	# ax2.set_ylabel('Training accuracy')
	# ax2.set_title('Training accuracy vs epochs')

	# ax3.plot(val_loss_each_epoch)
	# ax2.set_xlabel('Epochs')
	# ax3.set_ylabel('Validation loss')
	# ax3.set_title('Validation loss vs epochs')

	# ax4.plot(val_accuracy_each_epoch)
	# ax4.set_xlabel('Epochs')
	# ax4.set_ylabel('Validation accuracy')
	# ax4.set_title('Validation accuracy vs epochs')
	average = total_bac_output/count
	print(f"Average BAC 10-fold Cross Validation: {average}")

if __name__ == "__main__":
	main()<jupyter_output>Train Index: [  0   1   2   3   4   7   8   9  10  11  13  14  15  16  17  18  19  20
  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  38  39
  40  41  42  44  45  46  47  48  49  50  51  52  53  54  55  57  58  59
  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77
  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95
  96  97  98  99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
 114 115 116 117 118 119 121 122 123 125 126 128 129 130 131 132 133 135
 136 138 139 140 141 142 143 144 145 146 147 149 150 151 152 153 154 155
 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
 175 176 177 178 181 183 184 185 188 189 191 192 194 195 196 197 198 199
 200 201 202 203 204 206], size: 186
Test Index: [  5   6  12  37  43  56 120 124 127 134 137 148 156 179 180 182 186 187
 190 193 205], size: 21
Length of the dataset: 207
IMAGE SIZE : torch.Size([80, 51, 16, 16])
LABEL SIZE :  torch.Size([80, 1])
NETWORK [...]<jupyter_text><jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11281966490299823, 'fraction_numerical': 0.06437389770723104, 'mean_word_length': 3.425609756097561, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '408875', 'license': 'MIT', 'n_tokens_mistral': 7630, 'n_tokens_neox': 6749, 'n_words': 1479, 'path': 'Final_Classification_notebook.ipynb', 'repo_name': 'djramakrishna/melanoma', 'size': '18144'}