<jupyter_start><jupyter_text>Merge train and validation json file.<jupyter_code>import json
"""
train_json = "/home/ubuntu/data/complete_set/annotations/train.json"
test_json = "/home/ubuntu/data/complete_set/annotations/val.json"
complete_json = "/home/ubuntu/data/complete_set/annotations/complete.json"
"""
train_json = "/home/ubuntu/data/satellite/20190109/annotations/train.json"
test_json = "/home/ubuntu/data/satellite/20190109/annotations/val.json"
complete_json = "/home/ubuntu/data/satellite/20190109/annotations/complete.json"


# read json file
def read_json(json_file):
    with open(json_file) as f:
        return json.load(f)

train_info = read_json(train_json)
test_info = read_json(test_json)
print(len(train_info['annotations']))
print(len(test_info['annotations']))
print(train_info['images'][:2])
print(train_info['annotations'][:2])
print(test_info['images'][:2])
print(test_info['annotations'][:2])
# To merge these two json_files, we need to update the parts of annotations and images.
print("Update annatations part")
last_annotation_id = train_info['annotations'][-1]['id']
last_image_id = train_info['images'][-1]['id']
print(last_annotation_id)
print(last_image_id)
for annotation in test_info['annotations']:
    annotation['id'] += last_annotation_id
    annotation['image_id'] += last_image_id
    
    train_info['annotations'].append(annotation)

print("Update images parts")
for image in test_info['images']:
    image['id'] += last_image_id
    train_info['images'].append(image)
    
print(len(train_info['annotations']))
print(len(train_info['images']))


print(train_info['annotations'][-1]['id'])
print(train_info['images'][-1]['id'])
# save the complete json data
with open(complete_json, 'w') as f:
    json.dump(train_info, f)
# verify complete json file
with open(complete_json) as f:
    data_info = json.load(f)
    
print(len(data_info['annotations']))
print(len(data_info['categories']))
print(len(data_info['images']))<jupyter_output>129154
93
1625
{'fraction_non_alphanumeric': 0.156312625250501, 'fraction_numerical': 0.022044088176352707, 'mean_word_length': 6.956175298804781, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '32343', 'license': 'MIT', 'n_tokens_mistral': 697, 'n_tokens_neox': 650, 'n_words': 114, 'path': 'satellite/merge_json.ipynb', 'repo_name': 'gxdai/satellite', 'size': '1996'}