<jupyter_start><jupyter_text>Sample Frames from Videos, Save as JPG'sFor real frame sampling (pulling some subset of frames from video and saving as images), use the `./tools/extract_video_frames.py` script. The below cells are just for exploration.<jupyter_code>import cv2


def sample_frames(video_path: Path, out_path: Path, sample_every_seconds=20.0):
    """
    Sample a frame every `sample_every_seconds` seconds from the specified video, saving it to
    """
    path = video_path.with_suffix(".mp4")
    assert path.exists(), str(path)
    cap = cv2.VideoCapture(str(path))
    if not cap.isOpened():
        print("could not open :", video_path)
    num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    print(f"num_frames: {num_frames:,}")
    seconds = round(0, 2)
    sample_every_seconds = 5.0
    count = 0
    success = True
    print(path)
    while success:
        cap.set(cv2.CAP_PROP_POS_MSEC, (seconds * 1000))
        success, image = cap.read()
        if success:
            frame_out_path = out_path / "screen_samples_auto" / f"{path.stem}/frame_{count:08}.jpg"
            frame_out_path.parent.mkdir(exist_ok=True, parents=True)
            # print(frame_out_path)
            cv2.imwrite(str(frame_out_path), image)
        seconds = round(seconds + sample_every_seconds, 2)
        count += 1
    cap.release()
    print("total frames captured: ", count, ", seconds: ", seconds)


for i, (idx, row) in tqdm(enumerate(df_sampled.iterrows()), total=len(df_sampled)):
    # if i > 10:
    #     break
    sample_frames(row.path, OUT_PATH, 20.0)
df_sampled = df_geo.sample(100)
df_sampled<jupyter_output><empty_output><jupyter_text>--- ---  Split some categories<jupyter_code>imgs = data["images"]
imgs = {i["id"]: i for i in imgs}
imgs[0]
anns[0]
data = tasks
anns = data["annotations"]
cats = data["categories"]
cat_map = {c["name"]: c["id"] for c in cats}
for ann in data["annotations"]:
    if ann["category_id"] == cat_map["in_game_mini_map"]:
        bbox = ann["bbox"]
        area = bbox[2] * bbox[3]
        if area >= 120000:
            ann["category_id"] = cat_map["in_game_map_expanded"]
            print(ann)<jupyter_output>{'id': 50, 'image_id': 13, 'category_id': 24, 'segmentation': [], 'bbox': [425, 119, 832, 560], 'ignore': 0, 'iscrowd': 0, 'area': 465920}
{'id': 55, 'image_id': 14, 'category_id': 24, 'segmentation': [], 'bbox': [426, 68, 833, 612], 'ignore': 0, 'iscrowd': 0, 'area': 509796}
{'id': 76, 'image_id': 20, 'category_id': 24, 'segmentation': [], 'bbox': [698, 239, 562, 422], 'ignore': 0, 'iscrowd': 0, 'area': 237164}
{'id': 229, 'image_id': 57, 'category_id': 24, 'segmentation': [], 'bbox': [425, 136, 834, 541], 'ignore': 0, 'iscrowd': 0, 'area': 451194}
{'id': 259, 'image_id': 64, 'category_id': 24, 'segmentation': [], 'bbox': [698, 273, 563, 386], 'ignore': 0, 'iscrowd': 0, 'area': 217318}
{'id': 266, 'image_id': 66, 'category_id': 24, 'segmentation': [], 'bbox': [700, 273, 560, 386], 'ignore': 0, 'iscrowd': 0, 'area': 216160}
{'id': 912, 'image_id': 254, 'category_id': 24, 'segmentation': [], 'bbox': [473, 67, 785, 616], 'ignore': 0, 'iscrowd': 0, 'area': 483560}
{'id': 926, 'image_id': [...]<jupyter_text>---  Analysis for Customizing Anchor Box<jupyter_code>tasks = json.load(open(Path("../datasets/geoscreens_011/geoscreens_011.json")))
anns = tasks["annotations"]
Counter([img["width"] for img in tasks["images"]]), Counter(
    [img["height"] for img in tasks["images"]]
)
dims = [
    {
        "x": ann["bbox"][0],
        "y": ann["bbox"][1],
        "width": ann["bbox"][2],
        "height": ann["bbox"][3],
        "width_scaled": ann["bbox"][2] * (640 / 1280),
        "height_scaled": ann["bbox"][3] * (640 / 720),
        "area_scaled": (ann["bbox"][2] * (640 / 1280) * ann["bbox"][3] * (640 / 1280)),
        # "ratio_wh": float(ann["bbox"][2] * (640 / 1280)) / (ann["bbox"][3] * (640 / 1280)),
        "ratio_hw": float(ann["bbox"][3] * (640 / 1280)) / (ann["bbox"][2] * (640 / 1280)),
    }
    for ann in anns
]
df = pd.DataFrame(dims)
dims[:1]
df["area_bin"] = df.area_scaled.apply(
    lambda x: "small" if x <= 32 ** 2 else "medium" if x <= 96 ** 2 else "large"
)
df
anchor_sizes = tuple(
    (x, int(x * 2 ** (1.0 / 3)), int(x * 2 ** (2.0 / 3))) for x in [32, 64, 128, 256, 512]
)
aspect_ratios = ((0.08, 0.16, 0.25, 0.36, 0.5, 0.7, 1.0, 2.0),) * len(anchor_sizes)
anchor_sizes, aspect_ratios
pd.DataFrame(df.area_bin.value_counts()).plot.bar()
df.hist(column="width_scaled", bins=100, figsize=(48, 32), grid=True, by="area_bin")
df.hist(column="height_scaled", bins=100, figsize=(48, 32), grid=True, by="area_bin")
df.hist(column="ratio_hw", bins=100, figsize=(48, 32), grid=True, by="area_bin")
tuple((x, int(x * 2 ** (1.0 / 3)), int(x * 2 ** (2.0 / 3))) for x in [16, 32, 64, 128, 256])
anchor_sizes = tuple(
    (x, int(x * 2 ** (1.0 / 3)), int(x * 2 ** (2.0 / 3))) for x in [32, 64, 128, 256, 512]
)
aspect_ratios = ((0.5, 1.0, 2.0),) * len(anchor_sizes)
print(anchor_sizes, aspect_ratios)<jupyter_output>((32, 40, 50), (64, 80, 101), (128, 161, 203), (256, 322, 406), (512, 645, 812)) ((0.5, 1.0, 2.0), (0.5, 1.0, 2.0), (0.5, 1.0, 2.0), (0.5, 1.0, 2.0), (0.5, 1.0, 2.0))<jupyter_text>Train val Split<jupyter_code>from collections import Counter

Counter([Path(img["file_name"]).parent.name for img in coco["images"]])<jupyter_output><empty_output><jupyter_text>Load Tasks and Split the in_game_mini_map annotations into expanded / not expanded:<jupyter_code>tasks = json.load(open("/shared/gbiamby/geo/exports/geoscreens_006-proj_id_5_with_preds.json", "r"))
dims = []
for t in tasks:
    for a in t["annotations"]:
        # print(a["result"])

        dims.extend(
            [
                (v["value"]["x"], v["value"]["y"], v["value"]["width"], v["value"]["height"], v["value"]["rectanglelabels"][0])
                for v in a["result"]
                if (
                    (v["value"]["rectanglelabels"][0] == "in_game_mini_map")
                    or ("guess" in v["value"]["rectanglelabels"][0])
                )
            ]
        )

print(dims[0], len(dims))
print("max width: ", max([d[2] for d in dims]), ", min width: ", min([d[2] for d in dims]))
print("max height: ", max([d[3] for d in dims]), ", min height: ", min([d[3] for d in dims]))<jupyter_output>max width:  65.60000000000001 , min width:  1.3
max height:  85.47452256944443 , min height:  0.4444444444444444<jupyter_text>Based on result below, we'll use percent_of_total_image>=20% as the cutoff for dividing in_game_mini_map into expanded / not expanded<jupyter_code>areas = sorted(
    [(d[2] * d[3], 100.0 * d[2] * d[3] / (100.0 ** 2), d[4]) for d in dims], key=lambda x: x[1]
)
areas<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20285969615728328, 'fraction_numerical': 0.08445040214477212, 'mean_word_length': 3.8067287043664995, 'pattern_counts': {'":': 9, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '287639', 'license': 'MIT', 'n_tokens_mistral': 3009, 'n_tokens_neox': 2600, 'n_words': 629, 'path': 'notebooks/annotating.ipynb', 'repo_name': 'GiscardBiamby/geo', 'size': '6714'}