<jupyter_start><jupyter_text>**Creating function to import json file** Importing image id and annotations from json file<jupyter_code>import json
def json_to_dict(file):
    with open(file) as f:
        json_data = json.load(f)
        images = json_data['images']
        annotations = json_data['annotations']
        return(images,annotations)<jupyter_output><empty_output><jupyter_text>**Storing imported images and annotations**<jupyter_code>images, annotations = json_to_dict('./train.json')<jupyter_output><empty_output><jupyter_text>**Creating list of annotion and images id**<jupyter_code>list_annot = []
for annotation in annotations:
    annot = { "caption" : annotation["caption"], 
            "image_id" : annotation["image_id"],
             "id":annotation["id"]
            }
    list_annot.append(annot)<jupyter_output><empty_output><jupyter_text>**Function to convert into panda dataframe**<jupyter_code>import pandas as pd
def convert_to_df(dict_):
    return pd.DataFrame.from_dict(dict_)<jupyter_output><empty_output><jupyter_text>**Creating dataframe from the list**Calculating length of caption<jupyter_code>annot_df = convert_to_df(list_annot)
annot_df['caption_len'] = annot_df['caption'].apply(len)
annot_df.head()

len(annot_df)<jupyter_output><empty_output><jupyter_text>**Using Groupby method to group image id**<jupyter_code>grouped_single=annot_df.groupby('image_id')['caption_len'].apply(max).reset_index().rename(columns={'image_id':'image_id','caption_len' : 'caption_len'})


grouped_single.head()

len(grouped_single)<jupyter_output><empty_output><jupyter_text>**Merging dataframe**<jupyter_code>dfinal = grouped_single.merge(annot_df,on = ["image_id", "caption_len"], how="left")
dfinal.head()<jupyter_output><empty_output><jupyter_text>**Removing duplicates image id**<jupyter_code>dfinal = dfinal.drop_duplicates(subset=['image_id']).drop("caption_len",1)
dfinal.head()
dfinal=dfinal.reset_index()
len(dfinal) == len(images)<jupyter_output><empty_output><jupyter_text>**Converting dataframe into dictionary**<jupyter_code>annot_dict= dfinal.to_dict('r')
annot_dict[7740]
images[0]
len(annot_dict)
json_final = {
    "images": images,
    "annotations":annot_dict
}

with open('./captions_vizwiz2020.json', 'w') as json_file:
    json.dump(json_final, json_file)
#Saving top 200
 len(images[0:200]) == len(dfinal.head(200))
json_final_200 = {
    "images": images[0:200],
    "annotations":dfinal.head(200).to_dict('r')
}<jupyter_output>/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:1490: FutureWarning: Using short name for 'orient' is deprecated. Only the options: ('dict', list, 'series', 'split', 'records', 'index') will be used in a future version. Use one of the above to silence this warning.
  FutureWarning,<jupyter_text>**Dumping final 200 captions and image id**<jupyter_code>with open('./captions_vizwiz2020_top200.json', 'w') as json_file:
    json.dump(json_final_200, json_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1417058428475487, 'fraction_numerical': 0.016453995970449966, 'mean_word_length': 7.275, 'pattern_counts': {'":': 5, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '471544', 'license': 'MIT', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1030, 'n_words': 172, 'path': 'Construction of New Dataset/Vizwiz/Vizwiz.ipynb', 'repo_name': 'Meet088/Scrapping-images-from-instagram', 'size': '2978'}