<jupyter_start><jupyter_text>Celebrities:<jupyter_code>with open(azure_json_path, 'r') as f:
    celebrity_tracks_list = list(ijson.items(f, 'videos.item.insights.faces.item'))
df = json_normalize(celebrity_tracks_list)
df
df['source'] = 'microsoft'
segment_df = df.drop(['description', 'thumbnailId', 'referenceId', 'referenceType', 'title', 'thumbnails'], axis=1)\
               .rename(columns = {"imageUrl":"url"})
segment_df = segment_df.explode('instances').reset_index(drop=True)
segment_df['instances'].iloc[0]
def convert_to_secs(time_str):
    h, m, s = time_str.split(':')
    return int(h)*3600 + int(m)*60 + float(s)
segment_df['segment.start_time'] = segment_df['instances'].apply(lambda x: convert_to_secs(x['start']) if pd.notna(x) else x)
segment_df['segment.end_time'] = segment_df['instances'].apply(lambda x: convert_to_secs(x['end']) if pd.notna(x) else x)
segment_df.drop('instances', axis=1, inplace=True)
segment_df<jupyter_output><empty_output><jupyter_text>Logos:Output from 'brands' endpoint is not present. Labels:<jupyter_code>with open(azure_json_path, 'r') as f:
    labels_list = list(ijson.items(f, 'videos.item.insights.labels.item'))
df = json_normalize(labels_list)
df
df['source'] = 'microsoft'
df = df.explode('instances').reset_index(drop=True)
df = df.join(json_normalize(df['instances']))
df['segment.start_time'] = df['start'].apply(lambda x: convert_to_secs(x) if pd.notna(x) else x)
df['segment.end_time'] = df['end'].apply(lambda x: convert_to_secs(x) if pd.notna(x) else x)
df.drop(['instances', 'adjustedStart', 'adjustedEnd', 'start', 'end'], axis=1, inplace=True)
df<jupyter_output><empty_output><jupyter_text>Text/OCR:<jupyter_code>with open(azure_json_path, 'r') as f:
    ocr_list = list(ijson.items(f, 'videos.item.insights.ocr.item'))
df = json_normalize(ocr_list)
df
df['source'] = 'microsoft'
df = df.explode('instances').reset_index(drop=True)
df['segment.start_time'] = df['instances'].apply(lambda x: convert_to_secs(x['start']) if pd.notna(x) else x)
df['segment.end_time'] = df['instances'].apply(lambda x: convert_to_secs(x['end']) if pd.notna(x) else x)

def add_bounding_box(row):
    temp_dict = { 'left': row['left'], \
                  'right': row['left'] + row['width'], \
                  'top' : row['top'], \
                  'bottom' : row['top'] + row['height'] }
    return temp_dict
df['bounding_box'] = df.apply(add_bounding_box, axis=1)
df.drop(['instances', 'left', 'top', 'width', 'height'], axis=1, inplace=True)
df<jupyter_output><empty_output><jupyter_text>Speech:<jupyter_code>with open(azure_json_path, 'r') as f:
    transcript_list = list(ijson.items(f, 'videos.item.insights.transcript.item'))
df = json_normalize(transcript_list)
df
df['source'] = 'microsoft'
df = df.explode('instances').reset_index(drop=True)
df['segment.start_time'] = df['instances'].apply(lambda x: convert_to_secs(x['start']) if pd.notna(x) else x)
df['segment.end_time'] = df['instances'].apply(lambda x: convert_to_secs(x['end']) if pd.notna(x) else x)
df.drop(['instances', 'speakerId'], axis=1, inplace=True)
df<jupyter_output><empty_output><jupyter_text>Shots:<jupyter_code>with open(azure_json_path, 'r') as f:
    shot_list = list(ijson.items(f, 'videos.item.insights.shots.item'))
df = json_normalize(shot_list)
df
df['source'] = 'microsoft'
df = df.explode('instances').reset_index(drop=True)
df['segment.start_time'] = df['instances'].apply(lambda x: convert_to_secs(x['start']) if pd.notna(x) else x)
df['segment.end_time'] = df['instances'].apply(lambda x: convert_to_secs(x['end']) if pd.notna(x) else x)
def handle_keyframes(input_list):
    out_list = []
    for keyframe in input_list:
        instances = keyframe.pop('instances', None)
        if instances:
            keyframe['start_time'] = convert_to_secs(instances[0]['start'])
            keyframe['end_time'] = convert_to_secs(instances[0]['end'])
    return input_list
df['keyframes'] = df['keyFrames'].apply(handle_keyframes)
df.drop(['instances', 'keyFrames'], axis=1, inplace=True)
df<jupyter_output><empty_output><jupyter_text>Content Moderation:<jupyter_code>with open(azure_json_path, 'r') as f:
    moderation_list = list(ijson.items(f, 'videos.item.insights.visualContentModeration.item'))
df = json_normalize(moderation_list)
df
df['source'] = 'microsoft'
df = df.explode('instances').reset_index(drop=True)
df['segment.start_time'] = df['instances'].apply(lambda x: convert_to_secs(x['start']) if pd.notna(x) else x)
df['segment.end_time'] = df['instances'].apply(lambda x: convert_to_secs(x['end']) if pd.notna(x) else x)
df.drop('instances', axis=1, inplace=True)
df<jupyter_output><empty_output><jupyter_text>**Currently not using textualContentModeration, as there are no samples.**<jupyter_code>with open(azure_json_path, 'r') as f:
    moderation_list = list(ijson.items(f, 'videos.item.insights.textualContentModeration'))
df = json_normalize(moderation_list)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17750403877221324, 'fraction_numerical': 0.003432956381260097, 'mean_word_length': 6.837025316455696, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '569075', 'license': 'Apache-2.0', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1844, 'n_words': 302, 'path': 'Azure Media/Azure_Explorations.ipynb', 'repo_name': 'Rehan-Ahmar/exploring_vision_apis', 'size': '4952'}