<jupyter_start><jupyter_text>Google Colab Connect<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Import statements<jupyter_code>from sklearn.cluster import KMeans
import numpy as np
import cv2
import math
import scipy.spatial.distance as d


import os
import numpy as np
import matplotlib.pyplot as plt
import pickle
import pandas as pd 
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure 
import json
import datetime
import copy
from PIL import Image as im 
import joblib

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets, transforms, models
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
import scipy.cluster.vq as vq
from sklearn.cluster import KMeans
import numpy as np
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Saving and Loading function<jupyter_code># Saving and Loading models using joblib 
def save(filename, obj):
  with open(filename, 'wb') as handle:
      joblib.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)

def load(filename):
  with open(filename, 'rb') as handle:
      return joblib.load(filename)<jupyter_output><empty_output><jupyter_text>Data Loaders and data class<jupyter_code># Define transforms for the training data and testing data
train_transforms = transforms.Compose([transforms.RandomRotation(30),
                                       transforms.RandomSizedCrop(224),
                                       transforms.ToTensor(),
                                       transforms.Normalize([0.485, 0.456, 0.406],
                                                            [0.229, 0.224, 0.225])])
# custom class to provide the image name along with the image 
class ImageFolderWithPaths(datasets.ImageFolder):
    """Custom dataset that includes image file paths. Extends
    torchvision.datasets.ImageFolder
    """

    # override the __getitem__ method. this is the method that dataloader calls
    def __getitem__(self, index):
        # this is what ImageFolder normally returns 
        original_tuple = super(ImageFolderWithPaths, self).__getitem__(index)
        # the image name
        img_name = self.imgs[index][0].split("/")[-1]
        
        # print(path)
        # make a new tuple that includes original and the path
        tuple_with_path = (original_tuple + (img_name,))
        return tuple_with_path
# instantiate the dataset and dataloader
# Train 
# root directory of train , test and val needs to be put here  
train_data_dir = '/content/drive/MyDrive/SUB/PROJECTS/IR/myData'
train_dataset = ImageFolderWithPaths(train_data_dir, transform=train_transforms) 
batch = 64
train_dataloader = torch.utils.data.DataLoader(train_dataset ,batch_size=batch )
len(next(iter(train_dataloader)))<jupyter_output><empty_output><jupyter_text>CNN feature extractor class<jupyter_code># Class to extract features from images 
class Encoder(nn.Module):
    def __init__(self):
        super(Encoder, self).__init__()
        VGG = models.vgg16(pretrained=True)
        self.feature = VGG.features
        self.classifier = nn.Sequential(*list(VGG.classifier.children())[:-2])
        pretrained_dict = VGG.state_dict()
        model_dict = self.classifier.state_dict()
        pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
        model_dict.update(pretrained_dict)
        self.classifier.load_state_dict(model_dict)

        # self.pool = nn.MaxPool2d(2, stride=3) 
        self.cnn1 = nn.Conv2d(512, 250, 1)
        
    def forward(self, x):
        output = self.feature(x)
        output = self.cnn1(output)
        return output

extractor = Encoder()
print(extractor)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)<jupyter_output>cpu<jupyter_text>Feature Extraction and saving<jupyter_code># temp = []
# image_names = []
# extractor.to(device)

# count = 0
# for idx,loader in enumerate([train_dataloader]):
#   # labels got here are of no use 
#   print("--------------------doing------------")
#   for inputs, labels, paths in loader:
#     # print(inputs, labels, paths)
#     inputs = inputs.to(device)
#     feature_outputs = extractor(inputs)
  
#     for f, p in zip(feature_outputs, paths):
#       count = count + 1
#       print(count)
#       # specify your own path to save 
#       np.save( '/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/img_feaures/' + p , f.cpu().detach().numpy())
#       temp.append(f.cpu().detach().numpy())
#       image_names.append(p)

# All images features appended in temp and each of following shape 
# print(temp[0].shape) , print(len(image_names))
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/all_img_features.pkl",temp)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/all_img_names.pkl",image_names)<jupyter_output><empty_output><jupyter_text>Diving Images into train and test<jupyter_code># train_path = "/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/train/"
# test_path = "/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/test/"
# original_path = "/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/img_feaures/"
# import shutil 
# # moving in train 
# train_f = []
# train_name = []

# imgs = os.listdir(original_path)
# size = len(imgs)
# print(size)

# np.random.seed(0)
# np.random.shuffle(imgs)

# other = imgs[:int(1*size)]
# other = imgs[:4950]
# print(len(other))

# for img in other:
#   train_name.append(".".join(img.split('.')[:-1]))
#   train_f.append(np.load(original_path + img))
#   shutil.move(original_path + img, train_path) 

# # train images temp diff and names of images different 
# import shutil 
# # moving in test
# test_f = []
# test_name = []

# imgs = os.listdir(original_path)
# size = len(imgs)
# print(size)

# np.random.seed(0)
# np.random.shuffle(imgs)

# other = imgs[:int(1*size)]
# print(len(other))

# for img in other:
#   test_name.append(".".join(img.split('.')[:-1]))
#   test_f.append(np.load(original_path + img ))
#   shutil.move(original_path + img, test_path) 

  
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/train_name.pkl",train_name)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/train_f.pkl",train_f)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/test_name.pkl",test_name)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/test_f.pkl",test_f)<jupyter_output><empty_output><jupyter_text>Generating k-mean data<jupyter_code>train_f = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/train_f.pkl") 
train_name = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/train_name.pkl")
test_f = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/test_f.pkl") 
test_name = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/test_name.pkl")

# # With temp we create data points for kmeans 
# for idx,t in enumerate(train_f):
#   t_new = np.reshape(t,(250,7*7)).T
#   if idx == 0:
#     t_old = np.reshape(t,(250,7*7)).T
#   else:
#     t_old = np.append(t_old, t_new, 0)
# Data for k-means ready 
# t_old.shape 
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/k_mean_data.pkl", t_old)
t_old = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/k_mean_data.pkl")
# t_old.shape<jupyter_output><empty_output><jupyter_text>Applying K-mean<jupyter_code># kmeans = KMeans(n_clusters=2000, random_state=0).fit(t_old)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/k_mean_centers_2000.pkl", kmeans.cluster_centers_)
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/k_mean_onject_2000.pkl", kmeans)<jupyter_output><empty_output><jupyter_text>Generating Histograms and assignment maps<jupyter_code># Loading image features, image names , kmean centers for 1000
k_mean_centers = load('/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/k_mean_centers_250_1000.pkl')

def input_vector_encoder(feature, codebook):
    """
    Input all the local feature of the image
    Pooling (encoding) by codebook and return
    """
    code, _ = vq.vq(feature, codebook)
    word_hist, bin_edges = np.histogram(code, bins=range(codebook.shape[0] + 1))
    return word_hist

def get_assignment_map(feature, codebook):
    """
    Input all the local feature of the image
    Pooling (encoding) by codebook and return
    """
    code, _ = vq.vq(feature, codebook)
    return code
Hist_all = []

for idx,t in enumerate(train_f):
  t_new = np.reshape(t,(250,7*7)).T
  Hist_all.append(input_vector_encoder(t_new, k_mean_centers))
  
amap_all = []

for idx,t in enumerate(train_f):
  t_new = np.reshape(t,(250,7*7)).T 
  x = np.reshape(get_assignment_map(t_new, k_mean_centers),(7,7))
  amap_all.append(x)
  
# save('/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/Hist_all',Hist_all )
# np.array(Hist_all).shape
# save('/content/drive/MyDrive/SUB/PROJECTS/IR/ParisDataV2/CNN_250/others/assignment_map_250_1000.pkl',amap_all )

Hist_all = load('/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/Hist_all')<jupyter_output><empty_output><jupyter_text>All Images 64,64 dict with image : resized pixels<jupyter_code># !ls /content/drive/MyDrive/SUB/PROJECTS/IR/data/
# import cv2
# datapath = "/content/drive/MyDrive/SUB/PROJECTS/IR/data/"
# format = [".jpg",".png",".jpeg"]

# img_dict = {}

# num1 = 64
# num2 = 64
# count = 0
# path = os.path.join(datapath)

# for img in os.listdir(path):
#   if img.endswith(tuple(format)):
#     # print(img)
#     # train_name.append(img)
#     img2 = cv2.imread(os.path.join(path,img))
#     new_img = cv2.resize(img2, (num2, num1))
#     # train_img.append(new_img)
#     # print(new_img)
#     # print(img)
#     img_dict[img] = list(new_img)
#     count = count + 1
#     print(count)
#     # break
# save("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/data.pkl", img_dict)
img_dict = load("/content/drive/MyDrive/SUB/PROJECTS/IR/CNN_250/others/data.pkl")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13365843949044587, 'fraction_numerical': 0.02119824840764331, 'mean_word_length': 4.014471057884232, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '476542', 'license': 'MIT', 'n_tokens_mistral': 3796, 'n_tokens_neox': 3580, 'n_words': 661, 'path': 'DL_Bag_Of_Word_Model/preprocessing.ipynb', 'repo_name': 'AkankshaShrimal/Instance-Based-Image-Retrieval', 'size': '10048'}