<jupyter_start><jupyter_text>Auto crop and split objects into 4 sub-images**Pipeline:**1. Load image via coco api2. Get bounding box of objects3. Create 4 sub-images with object allocated to each corner4. Apply prediction to sub_images5. Save results to json format for further analysis6. Save sub_images to local storage with prediction results (label + bbox)- folder named as associated category- sub_images file name have suffix as _1, _2, _3, _4<jupyter_code># Load libraries
%matplotlib inline
from make_show_result import *
from pycocotools.coco import COCO
import numpy as np
import pandas as pd
import skimage.io as io
import matplotlib.pyplot as plt
import pylab
import torchvision.transforms.functional as TF
import PIL
import os
import json 
from urllib.request import urlretrieve
pylab.rcParams['figure.figsize'] = (8.0, 10.0)
# Define image directory
projectDir=os.getcwd()
dataDir='.'
dataType='val2017'
imageDir='{}/images/'.format(dataDir)
annFile='{}/images/{}_selected/annotations/instances_{}.json'.format(dataDir,dataType,dataType)<jupyter_output><empty_output><jupyter_text>1. Load image via coco api<jupyter_code># initialize COCO api for instance annotations
coco=COCO(annFile); print()

# display COCO categories 
cats = coco.loadCats(coco.getCatIds())
nms=[cat['name'] for cat in cats]
print('COCO categories: \n{}\n'.format(', '.join(nms)))

# display COCO supercategories
nms = set([cat['supercategory'] for cat in cats])
print('COCO supercategories: \n{}'.format(' '.join(nms)))<jupyter_output>loading annotations into memory...
Done (t=0.75s)
creating index...
index created!

COCO categories: 
person, bicycle, car, motorcycle, airplane, bus, train, truck, boat, traffic light, fire hydrant, stop sign, parking meter, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra, giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard, sports ball, kite, baseball bat, baseball glove, skateboard, surfboard, tennis racket, bottle, wine glass, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange, broccoli, carrot, hot dog, pizza, donut, cake, chair, couch, potted plant, bed, dining table, toilet, tv, laptop, mouse, remote, keyboard, cell phone, microwave, oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy bear, hair drier, toothbrush

COCO supercategories: 
accessory vehicle indoor sports electronic kitchen animal appliance food person furniture outdoor<jupyter_text>**Test loading 1 specific image / category**<jupyter_code>def load_coco_img(idNumber):
    """Returns a single image with corresponding ID.
    """
    imID = coco.getImgIds(imgIds = int(idNumber))
    imInfo = coco.loadImgs(imID)[0]
    I = io.imread(imInfo['coco_url'])
    return I

def load_coco_cat(catName):
    """Returns a list of image id of a single category.
    """
    catIds = coco.getCatIds(catNms=catName);
    imIds = coco.getImgIds(catIds=catIds)
    return imIds
# Test function load_coco_cat
imgIds_list = load_coco_cat("airplane")
print(imgIds_list)
# Test function load_coco_img
I = load_coco_img(1761)
plt.axis('off')
plt.imshow(I)
plt.show()<jupyter_output><empty_output><jupyter_text>2. Get bounding box of object<jupyter_code>def get_category_dictionary1():
    catLblFile='{}/images/val2017_selected/annotations/coco-labels-paper.txt'.format(dataDir)
    f = open(catLblFile, "r")
    category_name = list()
    for cat in f:
        category_name.append(cat)
    cat_dict = {category_name[i - 1].strip("\n"): i for i in range(1, len(category_name)+1)}
    return cat_dict

def get_category_dictionary2():
    catLblFile='{}/images/val2017_selected/annotations/coco-labels-paper.txt'.format(dataDir)
    f = open(catLblFile, "r")
    category_name = list()
    for cat in f:
        category_name.append(cat)
    cat_dict = {i: category_name[i - 1].strip("\n") for i in range(1, len(category_name)+1)}
    return cat_dict

def load_coco_anns(idNumber, catName):
    """Returns all annotation information of a specific image and associated category of objects existing in that image.
    """
    annIds = coco.getAnnIds(imgIds=idNumber, iscrowd=None)
    anns = coco.loadAnns(annIds)
    anns_cat = []
    catDict = get_category_dictionary1()
    for ann in anns:
        if (ann["category_id"] == catDict[catName]):
            anns_cat.append(ann)
    return anns_cat
print(get_category_dictionary1())
print(get_category_dictionary2())
# Test load_coco_anns on single image_id
selected_image_anns = load_coco_anns([1761], 'airplane')
print(selected_image_anns)

# Test load_coco_anns on list of image_id
# selected_image_anns = load_coco_anns(imgIds_list, 'airplane')
# print(selected_image_anns)<jupyter_output>[{'category_id': 5, 'iscrowd': 0, 'image_id': 1761, 'bbox': [282.23, 139.38, 51.22, 43.78], 'segmentation': [[282.23, 172.01, 291.32, 167.05, 295.04, 162.51, 292.97, 160.85, 295.86, 158.38, 299.99, 154.66, 296.28, 153.42, 301.65, 150.12, 309.49, 139.38, 305.78, 160.44, 307.43, 165.81, 309.08, 162.09, 319.41, 159.2, 328.49, 150.12, 328.9, 161.68, 333.45, 168.29, 324.36, 164.16, 321.47, 164.16, 308.25, 167.88, 306.6, 172.01, 318.58, 183.16, 309.49, 180.27, 303.71, 182.33, 305.78, 177.79, 303.3, 176.96, 297.93, 177.79, 301.65, 176.14, 296.69, 171.59, 284.3, 171.18]], 'area': 567.9410999999999, 'id': 159605}, {'category_id': 5, 'iscrowd': 0, 'image_id': 1761, 'bbox': [150.17, 8.63, 74.09, 77.19], 'segmentation': [[150.17, 65.64, 154.39, 61.87, 166.81, 58.98, 170.14, 47.0, 165.26, 48.56, 163.93, 47.0, 170.8, 43.01, 172.58, 32.81, 168.81, 34.58, 167.48, 32.81, 173.69, 28.59, 178.35, 11.73, 183.0, 8.63, 183.0, 52.11, 195.87, 47.23, 199.42, 46.56, 205.63, 45.23, 209.62, 31.03, 213.62, 28.37, 2[...]<jupyter_text>3. Create 4 sub-images 1. Create category folder if not exist2. Create image_id folder if not exist3. Crop image_1/2/3/44. Save image_1/2/3/45. Create summary.csv<jupyter_code>def create_category_folder(category):
    """Creates a folder named as category in ./images/ directory
       and returns a new path
    """
    imageDir='./images/'
    path = os.path.join(imageDir, category)
    try:
        if not (os.path.exists(path)):
            os.mkdir(path) 
    except OSError as error: 
        pass  
    return path

def create_imageID_folder(category_folder, image_id):
    path = os.path.join(category_folder, str(image_id))
    try: 
        if not (os.path.exists(path)):
            os.mkdir(path) 
    except OSError as error: 
        pass  
    return path
def download_coco_image(image_id, image_cat):
    """
    """
    # Create directory
    catDir = create_category_folder(image_cat)
    imgDir = create_imageID_folder(catDir, image_id)

    # Download image
    imgInfo = coco.loadImgs(image_id)[0]
    file_name = imgInfo['file_name']
    path = os.path.join(imgDir, imgInfo['file_name'])
    if not os.path.exists(path):
        coco.download(tarDir = imgDir, imgIds = [image_id])
    
    return path, imgDir
new_path, _ = download_coco_image(1761, 'airplane')
print(new_path)
def transform_bbox_coordinate(corner, bbox):
    corner_x, corner_y = corner.copy()
    x, y, w, h = map(round,bbox)
    new_x = (1 - corner_x) * x
    new_y = (1 - corner_y) * y
    return new_x, new_y, w, h

def transform_image_coordinate(corner, bbox, image_size):
    corner_x, corner_y = corner
    x, y, w, h = map(round,bbox)
    im_w, im_h = image_size
    new_x = corner_x * x
    new_y = corner_y * y
    new_h = corner_y * (im_h - y) + (1 - corner_y) * (y + h)
    new_w = corner_x * (im_w - x) + (1 - corner_x) * (x + w)
    return new_x, new_y, new_w, new_h

def generate_corner_objects(image_anns, return_image=False):
    """Reads info from 1 image's annotation and returns 2 parameters: 
    """
    catDict = get_category_dictionary2()
    imgDir, imgFolder = download_coco_image(
        image_anns['image_id'],
        catDict[image_anns['category_id']])
    img = PIL.Image.open(imgDir)

    # Crop sub-images
    corners = [[0,0], [0,1], [1,0], [1,1]]
    object_list = list()
    test_images = list()
    for idx, corner in enumerate(corners):
        _x, _y, _w, _h = transform_image_coordinate(corner, image_anns['bbox'], img.size)
        test_image = TF.crop(img, _y, _x, _h, _w)
        object_name = str(image_anns['id'])+'_'+str(idx)+'.jpg'
        test_image.save(os.path.join(imgFolder,object_name), "JPEG")
        test_images.append(test_image)

        #Create object_info
        object_info = dict()
        object_info['path'] = imgFolder
        object_info['filename'] = object_name
        object_info['bbox'] = list(transform_bbox_coordinate(corner, image_anns['bbox']))
        object_info['category'] = catDict[image_anns['category_id']]
        object_info['width'] = test_image.width
        object_info['height'] = test_image.height
        object_list.append(object_info)

    # Create object_id.csv file
    obj_df = pd.DataFrame(object_list)
    os.rename(imgDir, os.path.join(imgFolder, str(image_anns['id'])+'.jpg'))
    obj_df = obj_df.append({
        'bbox': list(map(int,image_anns['bbox'])),
        'category': catDict[image_anns['category_id']],
        'filename': str(image_anns['id'])+'.jpg',
        'path': imgFolder,
        'width': img.width,
        'height': img.height,
    }, ignore_index=True)

    obj_df.to_csv(os.path.join(imgFolder, str(image_anns['id']))+'.csv', header=True, index=False)
    
    if return_image:
        return test_images
    else:    
        return
selected_image_anns = load_coco_anns([1761], 'airplane') #1761
for ann in selected_image_anns:
    new_images = generate_corner_objects(ann, return_image=True)
    im1, im2, im3, im4 = new_images
    fig, axs = plt.subplots(1, 4)
    axs[0].imshow(np.asarray(im1))
    axs[1].imshow(np.asarray(im2))
    axs[2].imshow(np.asarray(im3))
    axs[3].imshow(np.asarray(im4))
    fig.tight_layout()
    plt.show()<jupyter_output>downloaded 0/1 images (t=1.5s)<jupyter_text>4. Automate cropping image<jupyter_code>def qualified_crop(obj_ann):
    x, y, w, h = map(round,obj_ann['bbox'])
    resolution_threshold = 28*28
    if w*h >= resolution_threshold:
        center_x, center_y = x+w/2, y+h/2
        imgInfo = coco.loadImgs(obj_ann['image_id'])[0]
        img_w, img_h = imgInfo['width'], imgInfo['height']
        # if (center_x >= img_w/3) and (center_x <= 2*img_w/3):
        #     if (center_y >= img_h/3) and (center_y <= 2*img_h/3):
        return True
    return False
# Load specific category
selected_category = "book"
imgIds_list = load_coco_cat(selected_category)

# Get all objects belong to that category
objAnns = load_coco_anns(imgIds_list, selected_category)
print('Total objects: {}'.format(len(objAnns)))

# Select certain number of object
selected_image_ids = []
for idx, ann in enumerate(objAnns):
    # print(ann)
    if len(selected_image_ids) >= 100:
        break
    if ann['image_id'] not in selected_image_ids:
        if qualified_crop(ann):
            generate_corner_objects(ann)
            selected_image_ids.append(ann['image_id'])<jupyter_output>Total objects: 1161
downloaded 0/1 images (t=2.5s)
downloaded 0/1 images (t=1.6s)
downloaded 0/1 images (t=1.4s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 images (t=1.5s)
downloaded 0/1 images (t=1.5s)
downloaded 0/1 images (t=1.5s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 images (t=1.8s)
downloaded 0/1 images (t=1.8s)
downloaded 0/1 images (t=1.0s)
downloaded 0/1 images (t=1.9s)
downloaded 0/1 images (t=1.5s)
downloaded 0/1 images (t=1.7s)
downloaded 0/1 images (t=1.4s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 images (t=1.7s)
downloaded 0/1 images (t=1.6s)
downloaded 0/1 images (t=1.8s)
downloaded 0/1 images (t=1.5s)
downloaded 0/1 images (t=1.8s)
downloaded 0/1 images (t=1.4s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 images (t=1.6s)
downloaded 0/1 images (t=1.4s)
downloaded 0/1 images (t=1.7s)
downloaded 0/1 images (t=2.3s)
downloaded 0/1 images (t=1.8s)
downloaded 0/1 images (t=1.2s)
downloaded 0/1 images (t=1.3s)
downloaded 0/1 imag[...]<jupyter_text>5. Gather all csv files<jupyter_code>all_files = []
for subdir, dirs, files in os.walk(os.path.join(imageDir,selected_category)):
    for filename in files:
        filepath = subdir + os.sep + filename
        if filepath.endswith(".csv"):
            all_files.append(filepath)
            print(filepath)
li = []
for filename in all_files:
    df = pd.read_csv(filename, index_col=None, header=0)
    li.append(df)
df_images = pd.concat(li, axis=0, ignore_index=True)
df_images.head(12)
df_images.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 500 entries, 0 to 499
Data columns (total 6 columns):
bbox        500 non-null object
category    500 non-null object
filename    500 non-null object
height      500 non-null int64
path        500 non-null object
width       500 non-null int64
dtypes: int64(2), object(4)
memory usage: 23.6+ KB
{'fraction_non_alphanumeric': 0.13618436315830176, 'fraction_numerical': 0.06372283659423092, 'mean_word_length': 3.9547892720306512, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '190027', 'license': 'MIT', 'n_tokens_mistral': 5317, 'n_tokens_neox': 4740, 'n_words': 1168, 'path': 'thesis_code/auto_crop.ipynb', 'repo_name': 'hhodac/keras-yolo3', 'size': '12931'}