<jupyter_start><jupyter_text>Importing necessary libraries<jupyter_code># import pandas as pd
import numpy as np
import os
# import seaborn as sns
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader, random_split
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder
from torchvision import models
from torch import optim
# import PIL
# import skimage
from torchvision.datasets.utils import download_url
# import tarfile
import zipfile
%matplotlib inline
np.random.seed = 42

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device<jupyter_output><empty_output><jupyter_text>Getting Kaggle datasets<jupyter_code>!pip install -q kaggle
!pip install -q kaggle-cli
!mkdir -p ~/.kaggle
!cp "/content/drive/MyDrive/Kaggle datasets/kaggle.json" ~/.kaggle/
!cat ~/.kaggle/kaggle.json 
!chmod 600 ~/.kaggle/kaggle.json
# For other datasets
!kaggle datasets download -d virenbr11/pothole-and-plain-rode-images -p Dataset
path_zip = '/content/Dataset/pothole-and-plain-rode-images.zip'
extract_to = '/content/Dataset/'

with zipfile.ZipFile(path_zip, 'r') as zip_ref:
    zip_ref.extractall(extract_to)<jupyter_output><empty_output><jupyter_text>Looking at dataset<jupyter_code>class_names = os.listdir('/content/Dataset/My Dataset/train')
class_names
path_train = '/content/Dataset/My Dataset/train'
path_test = '/content/Dataset/My Dataset/test'
def show_class(class_names, path):
  i = 1
  for cl in class_names:

     plt.subplot(1,len(class_names),i)
     pth = path + '/' + cl
     files = os.listdir(pth)
     im = plt.imread(pth+ '/' + files[0])
     plt.title(cl)
     plt.imshow(im)
     i = i+1
show_class(class_names, path_train)
show_class(class_names, path_test)<jupyter_output><empty_output><jupyter_text>ImageFolder Dataset<jupyter_code>batch_size = 32
transformations = transforms.Compose((transforms.Resize((224,224)),
                                      transforms.ToTensor(),
                                      transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])))
train_dataset = ImageFolder(path_train, 
                            transform = transformations)
test_dataset = ImageFolder(path_test,
                            transform = transformations)
train_loader = DataLoader(train_dataset,
                          batch_size = batch_size)
test_loader = DataLoader(test_dataset,
                          batch_size = batch_size)
# train_loader = train_loader.to(device)
# test_loader = test_loader.to(device)
for data in train_loader:
  x, y = data
  print(x.shape, y.shape)
  print(y)
  break
def show(img):
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
show(x[1]), y[1]
class_names.sort()
class_names_idx = dict(zip(list(np.arange(len(class_names))),class_names))<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># model = models.vgg16(pretrained= True)
model = models.resnet18(pretrained=True)

print(model)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, len(class_names))

# for param in model.parameters():
#     param.require_grad = False
# for param in model.fc.parameters():
#     param.require_grad = True

model = model.to(device)
optimizer = optim.SGD(model.fc.parameters(), lr = 0.001, momentum= 0.8)
criterion = nn.CrossEntropyLoss()
device<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>class train_model:

  def __init__(self, 
               train_loader,
               test_loader,
               task,
               model,
               optimizer,
               criterion,
               epochs):
    
    self.train_loader = train_loader
    self.test_loader = test_loader
    self.task = task
    self.optimizer = optimizer
    self.criterion = criterion
    self.epochs = epochs
    self.model = model

  

  def train_it(self):
    """
    """
    self.model.train()
    history = {'train_loss':[],
                 'test_loss':[],
                 'train_score':[],
                 'test_score':[]}
    for epoch in range(self.epochs):

      for loader in self.train_loader:

        self.optimizer.zero_grad()
        xs, ys = loader
        xs = xs.to(device)
        ys = ys.to(device)

        preds = self.predict(xs)
        loss = self.criterion(preds,ys.reshape(-1,).long())

        loss.backward()
        self.optimizer.step()
  
        
      self.model.eval()

      history['train_loss'].append(self.calc_loss(self.train_loader))
      history['test_loss'].append(self.calc_loss(self.test_loader))
      history['train_score'].append(self.calc_score(self.train_loader))
      history['test_score'].append(self.calc_score(self.test_loader))

      if epoch%1==0:
        print(f"Iteration : {epoch}")
        
        fig = plt.figure(figsize = (12,7))
        plt.subplot(1, 2, 1)
        plt.plot(history['train_loss'], color = 'red', label = 'Training Loss')
        plt.plot(history['test_loss'], color = 'green', label = 'Validation Loss')
        plt.legend()
        

        plt.subplot(1, 2, 2)
        plt.plot(history['train_score'], color = 'red', label = 'Training Score')
        plt.plot(history['test_score'], color = 'green', label = 'Validation Score' )
        plt.legend()
        plt.show()
        

        # fig = plt.figure()
        # ax = fig.add_subplot(111, projection = '3d')
        # ax.scatter(X[:,0], X[:,1], model(torch.tensor(X).float())[:,0].detach().numpy())
        # plt.show()
    return self.model, history

  def r2(self, xs, ys):
    return r2_score(ys.reshape(-1,), self.predict(xs).numpy())
 
  def acc(self, ys, preds):
    maxs, cls = torch.max(preds, axis = 1)
    
    return torch.sum(cls==ys.reshape(-1,)).item()/ys.shape[0]

  def predict(self,xs):
    return self.model(xs)

  def calc_loss(self,loader):
    loss = []
    for idx, data in enumerate(loader):
      xs, ys = data
      xs, ys = xs.to(device), ys.to(device)
      loss.append(self.criterion(self.predict(xs),ys.reshape(-1,).long()).item())
    return np.mean(loss)

  def calc_score(self,loader):
    scores = []
    for data in loader:
      xs, ys = data
      xs, ys = xs.to(device), ys.to(device)
      if self.task == 'reg':
          
          scores.append(self.r2(ys,self.predict(xs)))
          
      else:
          scores.append(self.acc(ys,self.predict(xs)))
    return np.mean(scores)
trainer = train_model(train_loader,
                      test_loader,
                      'clf',
                      model,
                      optimizer,
                      criterion,
                      4)
model, h = trainer.train_it()
# torch.cuda.empty_cache()<jupyter_output>/usr/local/lib/python3.6/dist-packages/PIL/Image.py:932: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images
  "Palette images with Transparency expressed in bytes should be "<jupyter_text>Testing on some images<jupyter_code>transforms_test = transforms.Compose((
                                      transforms.Resize((224,224)),
                                      transforms.ToTensor(),
                                      transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])))
test_path = '/content/nhai-asked-to-sort-out-delhi-gurgaon-expressway-project-issue-on-its-own.jpg'
plt.imshow(plt.imread(test_path))
import PIL.Image as Image
img = Image.open(test_path)  # Load image as PIL.Image
x = transforms_test(img).to(device)  # Preprocess image
x = x.unsqueeze(0)  # Add batch dimension

output = model(x)  # Forward pass
pred = torch.argmax(output, 1)  # Get predicted class if multi-class classification
print('Image predicted as ', class_names_idx[pred.item()])<jupyter_output>Image predicted as  Plain<jupyter_text>Interpretability<jupyter_code># !pip install shap
import shap<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12432498772704959, 'fraction_numerical': 0.015095729013254787, 'mean_word_length': 2.5663019693654268, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '116485', 'license': 'Apache-2.0', 'n_tokens_mistral': 2848, 'n_tokens_neox': 2667, 'n_words': 508, 'path': 'Image/pytorch_img_clf_pothole_detection.ipynb', 'repo_name': 'SPThole/pyTrch', 'size': '8148'}