<jupyter_start><jupyter_text>Query<jupyter_code>def show_query(video_id, antagonist_name):
    videos = Video.objects.filter(id__in=[video_id]).all()
    
    # Annotate face rows with start and end frames and the video ID
    faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        character_name=F('characteractor__character__name')
    ).filter(video_id__in=[v.id for v in videos])

    frames_with_identity = VideoIntervalCollection.from_django_qs(
        faces_with_character_actor_qs,
        with_payload=in_array(
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }),
        )
    ).coalesce(payload_merge_op=payload_plus)
    
    def convert_to_frames(intrvllist_seconds, fps):
        return intrvllist_seconds.map(
            lambda intrvl: (int(intrvl.start * fps), int(intrvl.end * fps), intrvl.payload)
        )
    
    all_segments = get_all_segments([video.id for video in videos])
    captions_interval_collection = VideoIntervalCollection({
        video: convert_to_frames(IntervalList(intervals), Video.objects.get(id=video).fps)
        for video, intervals in all_segments
    }).dilate(10).coalesce(payload_merge_op = payload_plus).dilate(-10).map(
        lambda intrvl: (intrvl.start, intrvl.end, ' '.join(intrvl.payload))
    )
    
    frames_with_antagonist = frames_with_identity.filter(
        lambda intrvl: any(antagonist_name in char['character'] for char in intrvl.payload)
    )
    
    frames_with_antagonist_and_text = captions_interval_collection.filter_against(
        frames_with_antagonist, predicate=overlaps(), working_window=10).coalesce().filter_length(min_length=48)
    
    result = intrvllists_to_result_with_objects(frames_with_antagonist_and_text, lambda a, b: [])
    widget = esper_widget(result, jupyter_keybindings=True)
    display(widget)
    return widget, result
def convert_segments(segments):
    output_segments = []
    for seg in segments:
        for res in result['result']:
            if res['elements'][0]['min_frame'] == seg[0]:
                output_segments.append((res['elements'][0]['min_frame'], res['elements'][0]['max_frame']))
    print(output_segments)
widget, result = show_query(28, 'longshanks')
selected_segments_braveheart = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['max_frame'])
    for i in widget.selected
]
print(selected_segments_braveheart)<jupyter_output>[(27910, 27973), (28084, 28298), (87826, 87933)]<jupyter_text>Revenge of the Sith<jupyter_code>start = time.time()
widget, result = show_query(186, 'palpatine')
selected_segments_rots = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['min_frame'])
    for i in widget.selected
]
print(selected_segments_rots)
selected_segments_rots = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['max_frame'])
    for i in widget.selected
]
print(selected_segments_rots)
end = time.time()
print("Seconds to label: ", end - start)<jupyter_output>Seconds to label:  375.02112674713135<jupyter_text><NAME><jupyter_code>start = time.time()
widget, result = show_query(520, 'sculley')
selected_segments_jobs = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['min_frame'])
    for i in widget.selected
]
print(selected_segments_jobs)
convert_segments(selected_segments_jobs)
end = time.time()
print("Seconds to label: ", end - start)<jupyter_output>Seconds to label:  298.2629656791687<jupyter_text>Guardians of the Galaxy<jupyter_code>start = time.time()
widget, result = show_query(74, 'ronan')
selected_segments_gotg = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['min_frame'])
    for i in widget.selected
]
print(selected_segments_gotg)
convert_segments(selected_segments_gotg)
end = time.time()
print("Seconds to label: ", end - start)<jupyter_output>Seconds to label:  112.39082598686218<jupyter_text>Daddy's Home<jupyter_code>start = time.time()
widget, result = show_query(334, 'dusty')
selected_segments_daddy = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['min_frame'])
    for i in widget.selected
]
print(selected_segments_daddy)
convert_segments(selected_segments_daddy)
end = time.time()
print("Seconds to label: ", end - start)<jupyter_output>Seconds to label:  272.71405601501465<jupyter_text>Batman v superman: no caption file<jupyter_code>def show_query_no_captions(video_id, antagonist_name):
# video_id=299
# antagonist_name="luthor"
# if True:
    videos = Video.objects.filter(id__in=[video_id]).all()
    
    # Annotate face rows with start and end frames and the video ID
    faces_with_character_actor_qs = FaceCharacterActor.objects.annotate(
        min_frame=F('face__frame__number'),
        max_frame=F('face__frame__number'),
        video_id=F('face__frame__video_id'),
        character_name=F('characteractor__character__name')
    ).filter(video_id__in=[v.id for v in videos])

    frames_with_identity = VideoIntervalCollection.from_django_qs(
        faces_with_character_actor_qs,
        with_payload=in_array(
            dict_payload_parser(VideoIntervalCollection.django_accessor, { 'character': 'character_name' }),
        )
    ).coalesce(payload_merge_op=payload_plus)
    
    def convert_to_frames(intrvllist_seconds, fps):
        return intrvllist_seconds.map(
            lambda intrvl: (int(intrvl.start * fps), int(intrvl.end * fps), intrvl.payload)
        )
    
    all_segments = get_all_segments([video.id for video in videos])
    captions_interval_collection = VideoIntervalCollection({
        video: convert_to_frames(IntervalList(intervals), Video.objects.get(id=video).fps)
        for video, intervals in all_segments
    }).dilate(10).coalesce(payload_merge_op = payload_plus).dilate(-10).map(
        lambda intrvl: (intrvl.start, intrvl.end, ' '.join(intrvl.payload))
    )
    
    frames_with_antagonist = frames_with_identity.filter(
        lambda intrvl: any(antagonist_name in char['character'] for char in intrvl.payload)
    )
    
    #print(frames_with_antagonist.get_intervallist(299))
    
    frames_with_antagonist_and_text = captions_interval_collection.filter_against(
        frames_with_antagonist, predicate=overlaps(), working_window=10).coalesce().filter_length(min_length=48)
    
    result = intrvllists_to_result_with_objects(
        frames_with_antagonist.dilate(10).coalesce().dilate(-10).filter_length(
            min_length=48
        ), lambda a, b: [])
    widget = esper_widget(result, jupyter_keybindings=True)
    display(widget)
    return widget, result
start = time.time()
widget, result = show_query_no_captions(299, 'luthor')
selected_segments_bvs = [
    (result['result'][i]['elements'][0]['min_frame'], result['result'][i]['elements'][0]['min_frame'])
    for i in widget.selected
]
print(selected_segments_bvs)
convert_segments(selected_segments_bvs)
end = time.time()
print("Seconds to label: ", end - start)<jupyter_output>Seconds to label:  456.8938891887665
{'fraction_non_alphanumeric': 0.12945693911135492, 'fraction_numerical': 0.023998902907295668, 'mean_word_length': 5.0978260869565215, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '237787', 'license': 'Apache-2.0', 'n_tokens_mistral': 2516, 'n_tokens_neox': 2424, 'n_words': 378, 'path': 'app/notebooks/trailer_mining_dark_tidings.ipynb', 'repo_name': 'DanFu09/esper', 'size': '7296'}