<jupyter_start><jupyter_text>Loading metadata:<jupyter_code>def load_caltech_images_from_json(json_filename):
    with open(json_filename) as json_file:
        cct_metadata = json.load(json_file)
    cct_images = pd.DataFrame(cct_metadata['images'])
    cct_annotations = pd.DataFrame(cct_metadata['annotations'])
    cct_images_labeled = pd.merge(cct_images,
                                  cct_annotations,
                                  how='outer',
                                  left_on='id',
                                  right_on='image_id')
    
    non_single_spc_instances = cct_images_labeled[
                                        cct_images_labeled[['image_id']].duplicated(keep=False)]
    non_single_spc_instances = non_single_spc_instances.image_id.unique()
    cct_images_labeled = cct_images_labeled[
                                ~cct_images_labeled.image_id.isin(non_single_spc_instances)].copy()
    
    return cct_images_labeled
with open('../data/CaltechCameraTrapsSplits_v0.json') as json_file:
    recommend_train_val_splits = json.load(json_file)
cct_images_all = load_caltech_images_from_json('../data/caltech_images_20200316.json')
cct_images_bbox = load_caltech_images_from_json('../data/caltech_bboxes_20200316.json')
cct_images_bbox['category_id'] = cct_images_bbox['category_id'].astype('int64')<jupyter_output><empty_output><jupyter_text>Initial split based on images with bbox:<jupyter_code>recommend_train = set(recommend_train_val_splits['splits']['train'])
recommend_val = set(recommend_train_val_splits['splits']['val'])
bbox_all_locations = set(cct_images_bbox.location.unique())
bbox_train = bbox_all_locations - recommend_val
bbox_val = bbox_all_locations - bbox_train
len(bbox_all_locations), len(bbox_train), len(bbox_val)
#bbox_val_dev = set(random.sample(bbox_train, 10))
bbox_val_dev = {'10', '115', '4', '52', '55', '56', '58', '59', '62', '73'}
bbox_train = bbox_train - bbox_val_dev<jupyter_output><empty_output><jupyter_text>Extrapolate split to all locations:<jupyter_code>val = recommend_val.copy()
train = bbox_train.copy()
val_dev = bbox_val_dev.copy()
remaining_train = recommend_train - train - val_dev
#remaining_val_dev = set(random.sample(remaining_train, 10))
remaining_val_dev = {'106', '107', '113', '118', '119', '122', '123', '137', '83', '96'}
remaining_train = remaining_train - remaining_val_dev
train = train.union(remaining_train)
val_dev = val_dev.union(remaining_val_dev)
len(train), len(val_dev), len(val)<jupyter_output><empty_output><jupyter_text>Visualize partitioning:<jupyter_code>def mark_split(train, val_dev, val, row):
    if row['location'] in val_dev:
        return 'val_dev'
    elif row['location'] in train:
        return 'train'
    else:
        return 'val'
marK_split_fn = partial(mark_split, train, val_dev, val)
instances = cct_images_bbox.copy()
instances['split'] = instances.apply(marK_split_fn, axis=1)
pd.crosstab(instances.category_id, instances.split)
def binarize_categories(row):
    if row['category_id'] == 30:
        return 0
    else:
        return 1
instances['category'] = instances.apply(binarize_categories, axis=1)
pd.crosstab(instances.category, instances.split)
instances_all = cct_images_all.copy()
instances_all['split'] = instances_all.apply(marK_split_fn, axis=1)
pd.crosstab(instances_all.category_id, instances_all.split)
instances_all['category'] = instances_all.apply(binarize_categories, axis=1)
pd.crosstab(instances_all.category, instances_all.split)<jupyter_output><empty_output><jupyter_text>Select empty images for training set:<jupyter_code>def cap_instances_by_location(instances, cap=1000):
    locations = instances.location.unique()
    instances_selected_list = []
    for location in locations:
        instances_per_location = instances[instances.location == location]
        if len(instances_per_location) > cap:
            instances_selected_list.append(instances_per_location.sample(cap))
        else:
            instances_selected_list.append(instances_per_location)
    return pd.concat(instances_selected_list)
noempty_bbox_locations = instances[instances.category==1].copy()
empty_all_locations = instances_all[instances_all.category==0].copy()
empty_train_capped = cap_instances_by_location(empty_all_locations[empty_all_locations.split=='train'])
empty_val_dev_capped = cap_instances_by_location(empty_all_locations[empty_all_locations.split=='val_dev'])
empty_val = empty_all_locations[empty_all_locations.split=='val'].copy()
selected_instances = pd.concat([noempty_bbox_locations, empty_train_capped, empty_val_dev_capped, empty_val])
pd.crosstab(selected_instances.category, selected_instances.split)<jupyter_output><empty_output><jupyter_text>Saving data:<jupyter_code>caltech_split_v3 = {'train': list(train), 'val_dev': list(val_dev), 'val': list(val)}
with open('../data/caltech_splits_v3.json', 'w') as outfile:
    json.dump(caltech_split_v3, outfile, indent=2)
with open('../data/caltech_images_20200316.json') as json_file:
    all_images = json.load(json_file)
all_images['info']['version'] = '20201110ufam'

instances_2_json = selected_instances[['seq_num_frames',
                                        'date_captured',
                                        'seq_id',
                                        'height',
                                        'width',
                                        'location',
                                        'rights_holder',
                                        'file_name',
                                        'id_x',
                                        'frame_num']]
instances_2_json = instances_2_json.rename(columns={'id_x': 'id'})
instances_2_json = instances_2_json.to_dict('records')
all_images['images'] = instances_2_json
with open('../data/caltech_images_20201110ufam.json', 'w') as outfile_images:
    json.dump(all_images, outfile_images, indent=2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11533928270752652, 'fraction_numerical': 0.020205421788179828, 'mean_word_length': 4.016891891891892, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '664032', 'license': 'Apache-2.0', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1934, 'n_words': 265, 'path': 'notebooks/cct_split_v3.ipynb', 'repo_name': 'alcunha/filtering-empty-camera-trap-images', 'size': '5939'}