<jupyter_start><jupyter_text>SUMMARYfor cleaned full captions dataset, do clustering on their hashtags<jupyter_code>import os
import json

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

%matplotlib inline
%load_ext autoreload
%autoreload 2
HASHTAG_DIR = '/media/alex/Project/MadeWithML_Incubator/data/instagram/json'
HASHTAG_JSONS = ['insta-hashtag-test1.json', 'insta-hashtag-test2.json', 'insta-hashtag-train.json']
INPUT_JSON = './ig_json/full_clean.json'

assert os.path.isdir(HASHTAG_DIR)<jupyter_output><empty_output><jupyter_text>1. Build Hashtag Mapping<jupyter_code>def read_json(json_path):
    assert os.path.isfile(json_path)
    with open(json_path, 'r') as f:
        data = json.load(f)
    return data


def create_hashtag_map(json_path):    
    hashtag_data = read_json(json_path)
    
    hashtag_map = dict()
    for usr_key, hist_dict in hashtag_data.items():
        for post_key, post_dict in hist_dict.items():
            id_key = f'{usr_key}_@_{post_key}'
            tags = post_dict['tags']
            hashtag_map[id_key] = tags
    return hashtag_map
hashtag_map = dict()
for hashtag_fn in HASHTAG_JSONS:
    json_path = f'{HASHTAG_DIR}/{hashtag_fn}'
    tmp_map = create_hashtag_map(json_path)
    hashtag_map.update(tmp_map)
    print(f'processed: {json_path}')<jupyter_output>processed: /media/alex/Project/MadeWithML_Incubator/data/instagram/json/insta-hashtag-test1.json
processed: /media/alex/Project/MadeWithML_Incubator/data/instagram/json/insta-hashtag-test2.json
processed: /media/alex/Project/MadeWithML_Incubator/data/instagram/json/insta-hashtag-train.json<jupyter_text>2. Compare with Input Data<jupyter_code>input_data = read_json(INPUT_JSON)

input_map = dict()
for caption_dict in input_data['images']:
    id_key = caption_dict['filename']
    captions = caption_dict['sentences'][0]['tokens']
    input_map[id_key] = captions
len(input_map)
miss = 0
for id_key, captions in input_map.items():
    if id_key not in hashtag_map:
        miss += 1

print(f'total input: {len(input_map)}')
print(f'miss hashtag: {miss}')<jupyter_output>total input: 469148
miss hashtag: 398385
{'fraction_non_alphanumeric': 0.11276794035414725, 'fraction_numerical': 0.010251630941286114, 'mean_word_length': 4.930939226519337, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '576518', 'license': 'MIT', 'n_tokens_mistral': 852, 'n_tokens_neox': 804, 'n_words': 141, 'path': 'data/notebooks/03_hashtag_analysis.ipynb', 'repo_name': 'riven314/Stylised-Controllable-Image-Captioning', 'size': '2146'}