<jupyter_start><jupyter_text>Connecting to Drive<jupyter_code>from google.colab import drive
drive.mount('/gdrive')<jupyter_output>Mounted at /gdrive<jupyter_text>Imports<jupyter_code>import re
import os
import sys
import cv2
import time
import json
import string
import random
import warnings
import argparse
import progressbar
import numpy as np
import pandas as pd

from tqdm.notebook import tqdm

from fastai.vision import *
from fastai.basic_data import *
from fastai.metrics import accuracy
from fastai.callbacks.hooks import num_features_model

import torch<jupyter_output><empty_output><jupyter_text>Copying Data/Model<jupyter_code>!mkdir data
!mkdir models

!cp '/gdrive/My Drive/Semester 8/CV/snek/datasets/train_small_10.tar.gz' ./data
!cp '/gdrive/My Drive/Semester 8/CV/snek/models/snake-detection-model.pt' ./models

!tar -xvf ./data/train_small_10.tar.gz -C ./data/
MODEL_PATH = "./models/snake-detection-model.pt"
DATASET_PATH = "./data/train/"
OUTPUT_PATH = "./data/output/"<jupyter_output><empty_output><jupyter_text>Define Model and Datasets<jupyter_code>class SnakeDetector(nn.Module):
    def __init__(self, arch=models.resnet18):
        super().__init__() 
        self.cnn = create_body(arch)
        self.head = create_head(num_features_model(self.cnn) * 2, 4)
        
    def forward(self, im):
        x = self.cnn(im)
        x = self.head(x)
        return x.sigmoid_()
class CustomDataset(Dataset):
    def __init__(self, j, aug=None):
        self.j = j
        if aug is not None: aug = get_aug(aug)
        self.aug = aug
    
    def __getitem__(self, idx):
        item = j2anno(self.j[idx])
        if self.aug: item = self.aug(**item)
        im, bbox = item['image'], np.array(item['bboxes'][0])
        im, bbox = self.normalize_im(im), self.normalize_bbox(bbox)
        
        return im.transpose(2,0,1).astype(np.float32), bbox.astype(np.float32)
    
    def __len__(self):
        return len(self.j)
    
    def normalize_im(self, ary):
        return ((ary / 255 - imagenet_stats[0]) / imagenet_stats[1])
    
    def normalize_bbox(self, bbox):
        return bbox / SZ<jupyter_output><empty_output><jupyter_text>Initialize Model<jupyter_code>src = (ImageList.from_folder(path=DATASET_PATH).split_by_rand_pct(0.0).label_from_folder())
tfms = get_transforms(do_flip=True,flip_vert=False,max_rotate=10.0,max_zoom=1.1,max_lighting=0.2,max_warp=0.2,p_affine=0.75,p_lighting=0.75)
data = (src.transform(tfms, size=360, resize_method=ResizeMethod.SQUISH)
        .databunch(bs=32).normalize(imagenet_stats))
learn = Learner(data, SnakeDetector(arch=models.resnet50), loss_func=torch.nn.L1Loss())
learn.split([learn.model.cnn[:6], learn.model.cnn[6:], learn.model.head])

state_dict = torch.load(MODEL_PATH)
learn.model.load_state_dict(state_dict['model'])<jupyter_output><empty_output><jupyter_text>Crop Images<jupyter_code>src_new = (ImageList.from_folder(path=DATASET_PATH).split_by_rand_pct(0.0).label_from_folder())
str_name = str(src_new.items[0])

with warnings.catch_warnings():
    warnings.simplefilter("ignore")

    for filename in tqdm(src_new.items):
        try:
            start = time.time()
            
            im = cv2.imread(f"{filename}", cv2.IMREAD_COLOR)
            im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
            im = cv2.resize(im, (360,360), interpolation = cv2.INTER_AREA)
            im_height, im_width, _ = im.shape
            
            orig_im = cv2.imread(f"{filename}", cv2.IMREAD_COLOR)
            orig_im_height, orig_im_width, _ = orig_im.shape
            to_pred = open_image(filename)
            
            _,_,bbox=learn.predict(to_pred)
            
            im_original = cv2.imread(f"{filename}", cv2.IMREAD_COLOR)
            im_original = cv2.cvtColor(im_original, cv2.COLOR_BGR2RGB)
            im_original.shape
            im_original_width = im_original.shape[1]
            im_original_height = im_original.shape[0]
            
            bbox_new = bbox
            bbox_new[0] = bbox_new[0]*im_original_width 
            bbox_new[2]= bbox_new[2]*im_original_width
            bbox_new[1] = bbox_new[1]*im_original_height
            bbox_new[3] = bbox_new[3]*im_original_height
            x_min, y_min, x_max, y_max = map(int, bbox_new)
            
            im_original = im_original[y_min:y_max,x_min:x_max]
            im_original = cv2.cvtColor(im_original,cv2.COLOR_BGR2RGB)
            filename_str = str(filename)
            
            to_save = filename_str.replace('train','cropped_images')
            to_save = to_save.split("/")
            file_name = "/".join(to_save[len(to_save)-2:])
            class_id = OUTPUT_PATH + to_save[-2]
            
            if not os.path.exists(class_id):
                os.makedirs(class_id)
            
            to_save = OUTPUT_PATH + file_name
            cv2.imwrite(to_save, im_original)
            # print("saved", to_save)
            # print('It took', time.time()-start, 'seconds.')
        except Exception as e:
            print(str(e))<jupyter_output><empty_output><jupyter_text>Save Data<jupyter_code>!for name in ./data/outputclass*; do newname="$(echo "$name" | cut -c14-)"; mv "$name" "$newname"; done
!mkdir ./data/train_cropped
!mv class* ./data/train_cropped
!tar -cvzf train_small_10_cropped.tar.gz ./data/train_cropped
!cp train_small_10_cropped.tar.gz '/gdrive/My Drive/Semester 8/CV/snek/datasets/'
!du -sh train_small_10_cropped.tar.gz

!find ./data/train/ -type f | wc -l
!find ./data/train_cropped/ -type f | wc -l<jupyter_output>1.5G	train_small_10_cropped.tar.gz
15014
14981<jupyter_text>33 images not cropped because error<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12356979405034325, 'fraction_numerical': 0.020418940327407147, 'mean_word_length': 3.1626373626373625, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '27144', 'license': 'MIT', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2064, 'n_words': 329, 'path': 'notebooks/crop_images.ipynb', 'repo_name': 'Aniket-Pradhan/cv_snek', 'size': '5681'}