<jupyter_start><jupyter_text>Simply Convert Data to COCO Format- Split the training data to train and validation- Convert both to COCO Formatted JSON<jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
# Some basic setup:
# import some common libraries
import numpy as np
import pandas as pd
import os
import json
import random
import cv2
import matplotlib.pyplot as plt
import ast
from tqdm import tqdm
from sklearn.model_selection import train_test_split
TOP_INPUT_DIR = "/content/drive/My Drive/GWD/global-wheat-detection/"
DATA_TRAIN_DIR = f"{TOP_INPUT_DIR}train/"
DATA_TEST_DIR = f"{TOP_INPUT_DIR}test/"<jupyter_output><empty_output><jupyter_text>Create COCO Base<jupyter_code>coco_base = { "info": {},
              "licenses": [], 
              "images": [],
              "annotations": [],
              "categories": []}<jupyter_output><empty_output><jupyter_text>Write dataset info to COCO Format<jupyter_code>coco_base["info"] = {
    "description": "Global Wheat Detection Dataset, Kaggle 2020",
    "url": "https://www.kaggle.com/c/global-wheat-detection/data",
    "version": "1.0",
    "year": 2020,
    "contributor": "http://www.global-wheat.com/contributors/",
    "date_created": "2020/05/29"
}<jupyter_output><empty_output><jupyter_text>Write licenses to COCO Format<jupyter_code>coco_base["licenses"].append(
    {
        "url": "https://opensource.org/licenses/MIT",
        "id": 1,
        "name": "MIT License"
    }
)<jupyter_output><empty_output><jupyter_text>Write category to COCO Format<jupyter_code>coco_base["categories"].append({"supercategory": "grain","id": 1,"name": "wheat"})
train_df = pd.read_csv(f'{TOP_INPUT_DIR}train.csv')
train_df.head()
uniq_images = train_df.image_id.unique()
len(uniq_images)<jupyter_output><empty_output><jupyter_text>There are 3373 unique images in the training set<jupyter_code>print(train_df.width.unique()[0])
print(train_df.height.unique()[0])<jupyter_output>1024
1024<jupyter_text>All images are of size 1024x1024  Group bboxes by imageCreate one row per image - combining all bboxes for that image into one column. This helps significantly reduce the number of rows to traverse.<jupyter_code>def get_bboxes_per_image(df):
    """author: @impiyush"""
    # convert from string list, to python list
    df.bbox = df.bbox.apply(ast.literal_eval)
    # group-by on image id and return all bboxes for that image as a list of lists
    return train_df.groupby('image_id')['bbox'].apply(list).reset_index(name='bboxes')
train_df_bboxes_grped = get_bboxes_per_image(train_df)
train_df_bboxes_grped.head()
assert train_df_bboxes_grped.shape[0] == len(uniq_images), "Number of images differ when grouped"<jupyter_output><empty_output><jupyter_text>Split to train and validation<jupyter_code>df_train, df_valid = train_test_split(
    train_df_bboxes_grped,
    test_size=0.05,
    random_state=32,
    shuffle=True
)

df_train = df_train.reset_index(drop=True)
df_valid = df_valid.reset_index(drop=True)
print(df_train.shape[0], df_valid.shape[0])<jupyter_output>3204 169<jupyter_text>Write images to COCO Format<jupyter_code>coco_base_train = coco_base.copy()
coco_base_valid = coco_base.copy()
def set_coco_images(df):
    """author: @impiyush"""
    images = []
    for _,img in enumerate(tqdm(df.image_id.unique())):
        img_dict = {"license":1, 
                    "height":1024, 
                    "width":1024,
                    "id":img}
        img_dict["file_name"] = f"{img}.jpg"
        images.append(img_dict)

    return images
coco_base_train["images"] = set_coco_images(df_train)
coco_base_valid["images"] = set_coco_images(df_valid)<jupyter_output>100%|██████████| 3204/3204 [00:00<00:00, 186602.47it/s]
100%|██████████| 169/169 [00:00<00:00, 314061.75it/s]<jupyter_text>Let's check the first three images in the list<jupyter_code>coco_base_train["images"][:3]
coco_base_valid["images"][:3]<jupyter_output><empty_output><jupyter_text>Let's also verify the number of images match, just for sanity<jupyter_code>assert len(coco_base_train["images"])==len(df_train), "Number of images differ from df_train"
assert len(coco_base_valid["images"])==len(df_valid), "Number of images differ from df_valid"<jupyter_output><empty_output><jupyter_text>Write annotations to COCO FormatThe most important piece of this puzzle<jupyter_code>def set_coco_annotations(df):
    """author: @impiyush"""
    annos = []
    id_cnt = 1

    for _,row in tqdm(df.iterrows(), total=len(df)):
    #     print(row)
        anno = {}
        anno['segmentation'] = []
        anno['iscrowd'] = 0
        anno['image_id'] = row['image_id']
        anno['category_id'] = 1
        bboxes = row['bboxes']
    #     annos.append(box)
        for ix, box in enumerate(bboxes):
            anno['bbox'] = box # x,y,w,h
            anno['area'] = box[2] * box[3] # w*h
            anno['id'] = f"{id_cnt:05}"
            annos.append(anno.copy()) # copy is necessary here, otherwise it will always point to the last value of anno
            id_cnt += 1
    
    return annos
coco_base_train['annotations'] = set_coco_annotations(df_train)
coco_base_valid['annotations'] = set_coco_annotations(df_valid)<jupyter_output>100%|██████████| 3204/3204 [00:00<00:00, 4443.25it/s]
100%|██████████| 169/169 [00:00<00:00, 4264.86it/s]<jupyter_text>Dump COCO formatted JSON for train and validation sets<jupyter_code>with open(f'/content/drive/My Drive/GWD/coco_train.json','w') as train_coco:
    json.dump(coco_base_train, train_coco)
with open(f'/content/drive/My Drive/GWD/coco_valid.json','w') as valid_coco:
    json.dump(coco_base_valid, valid_coco)
with open(f'/content/drive/My Drive/GWD/coco_test.json','w') as valid_cocoс:
    json.dump(None, valid_cocoс)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13829602637515184, 'fraction_numerical': 0.03071317022384175, 'mean_word_length': 4.76976976976977, 'pattern_counts': {'":': 21, '<': 53, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '65349', 'license': 'MIT', 'n_tokens_mistral': 2240, 'n_tokens_neox': 2064, 'n_words': 402, 'path': 'simply_convert_data_to_coco_format.ipynb', 'repo_name': 'germanjke/Global_Wheat_Detection', 'size': '5763'}