<jupyter_start><jupyter_text>Convert annot. file to COCO format & data split  Imports<jupyter_code>import os 
from glob import glob
import sys
import json
import cv2

FRAMES_DIR = os.path.join("data","frames")
video_name = "top-100-shots-rallies-2018-atp-season.mp4"
video_file = os.path.join("data",video_name)<jupyter_output><empty_output><jupyter_text>COCO template<jupyter_code># https://towardsdatascience.com/getting-started-with-coco-dataset-82def99fa0b8
COCO_JSON_CONTENT_TEMPLATE = '''
  "info": {info_dict},
  "licenses": {licenses_list},
  "images": {images_list},
  "categories": {categories_list},
  "annotations": {annotations_list}'''

IMAGE_DICT_CONTENT_TEMPLATE = '''
    "id": {id}, 
    "width": {width}, 
    "height": {height}, 
    "file_name": "{file_name}", 
    "license": {license}'''

BBOX_ANNOT_DICT_CONTENT_TEMPLATE = '''
    "image_id": {image_id},
    "bbox":
    [
        {x1},
        {y1},
        {width},
        {height}
    ],
    "category_id": {category_id},
    "id": {id}, 
    "iscrowd":0,
    "area": {area}
'''
coco = COCO_JSON_CONTENT_TEMPLATE<jupyter_output><empty_output><jupyter_text>Read orig. annot. and number of extracted frames, image dims.<jupyter_code>with open(os.path.join("data","top-100-shots-rallies-2018-atp-season-scoreboard-annotations.json")) as json_file:
    data = json.load(json_file)
frames_n = len(glob(os.path.join(FRAMES_DIR,"*.jpg")))
print(f"Frames number: {frames_n}")
img = cv2.imread(os.path.join(FRAMES_DIR,"0.jpg"))
height, width, _ = img.shape
print(f"Image dimensions: {height} x {width}")<jupyter_output>Image dimensions: 1080 x 1920<jupyter_text>Read video info<jupyter_code>video = cv2.VideoCapture(video_file)
video_fps = video.get(cv2.CAP_PROP_FPS)
video_length = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
print(f"{video_file}: {video_length} frames at {video_fps} FPS")<jupyter_output>data/top-100-shots-rallies-2018-atp-season.mp4: 43247 frames at 25.0 FPS<jupyter_text>Data split!!<jupyter_code>times = ["17:47","22:17"]
times = [int(t.split(":")[0])*60+int(t.split(":")[1]) for t in times]
t0, t1 = times
train = [0, int(video_fps*t0)]
val = [int(video_fps*t0), int(video_fps*t1)]
test = [int(video_fps*t1), frames_n]
print(train, val, test)
splits = ["train", "val", "test"]
splits_frame_limits = [train, val, test]<jupyter_output>[0, 26675] [26675, 33425] [33425, 43245]<jupyter_text>Writa data to template  Basic data<jupyter_code>licenses_list = [
  {
    "id": 1,
    "name": "TODO",
    "url": "TODO",  
  }
]

categories_list = [{"supercategory": "scoreboard", "id": 1, "name": "scoreboard"}]<jupyter_output><empty_output><jupyter_text>Build images list<jupyter_code>images_lists = []
for split, split_limits in zip(splits, splits_frame_limits):
    images_list = []
    for frame_idx in range(split_limits[0], split_limits[1]):
        filename = f"{frame_idx}.jpg"
        frame_filepath = os.path.join(FRAMES_DIR,filename)
        assert os.path.exists(frame_filepath)
        img_data = json.loads('{'+IMAGE_DICT_CONTENT_TEMPLATE.format(id=frame_idx, 
                                                      width=width,
                                                      height=height,
                                                      file_name=filename, 
                                                      license=1)+'}')
        
        images_list.append(img_data)
    images_lists.append(images_list)<jupyter_output><empty_output><jupyter_text>Build annotations list<jupyter_code>annotations_lists = [[] for split in splits]
for frame_idx, record in data.items():
    frame_idx = str(frame_idx)
    x1, y1, x2, y2 = record["bbox"]
    x, y = x1, y1
    assert x2 >= x1 
    assert y2 >= y1
    w, h = x2-x1, y2-y1
    image_id = frame_idx
    anno_data = json.loads('{'+BBOX_ANNOT_DICT_CONTENT_TEMPLATE.format(image_id = frame_idx, 
                                                                  x1 = x,
                                                                  y1 = y, 
                                                                  width = w, 
                                                                  height = h,
                                                                  category_id = 1,
                                                                  id = frame_idx,
                                                                  area = w*h)+'}')
    # add custom keys
    for split_idx in range(len(splits)):
        low, high = splits_frame_limits[split_idx]
        if low <= int(frame_idx) < high:
            annotations_lists[split_idx].append(anno_data)
assert sum(len(l) for l in annotations_lists) == len(data.keys())<jupyter_output><empty_output><jupyter_text>Build COCO dict.<jupyter_code>for split, images_list, annotations_list in zip(splits, images_lists, annotations_lists):
    coco = json.loads('{'+COCO_JSON_CONTENT_TEMPLATE.format(info_dict = {"description": "Sportsradar interview task {}".format(split)},
                                                     licenses_list = licenses_list,
                                                     images_list = images_list,
                                                     categories_list = categories_list,
                                                     annotations_list = annotations_list).replace("\'","\"")+'}')
    assert len(coco["images"]) == len(images_list)
    assert len(coco["annotations"]) == len(annotations_list)
    dest_fp = os.path.join("data",
                       f"top-100-shots-rallies-2018-atp-season-scoreboard-annotations_coco_{split}.json")
    with open(dest_fp, 
              'w') as f:
        json.dump(coco, f)
        print(f"Saved COCO-style dict. to {dest_fp}")<jupyter_output>Saved COCO-style dict. to data/top-100-shots-rallies-2018-atp-season-scoreboard-annotations_coco_train.json
Saved COCO-style dict. to data/top-100-shots-rallies-2018-atp-season-scoreboard-annotations_coco_val.json
Saved COCO-style dict. to data/top-100-shots-rallies-2018-atp-season-scoreboard-annotations_coco_test.json
{'fraction_non_alphanumeric': 0.13056379821958458, 'fraction_numerical': 0.02456313880646225, 'mean_word_length': 2.43350311262026, 'pattern_counts': {'":': 25, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '367456', 'license': 'Apache-2.0', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1964, 'n_words': 332, 'path': 'preprocessing_ann.ipynb', 'repo_name': 'igor-morawski/tennis-SR', 'size': '6066'}