<jupyter_start><jupyter_text>Load Shot Data<jupyter_code>with open('../../data/shot_detection_folds.pkl', 'rb') as f:
    shot_detection_folds = pickle.load(f)
with open('../../data/manually_annotated_shots.pkl', 'rb') as f:
    shots = VideoIntervalCollection(pickle.load(f))
clips = shots.dilate(1).coalesce().dilate(-1)
shot_boundaries = shots.map(
    lambda intrvl: (intrvl.start, intrvl.start, intrvl.payload)
).set_union(
    shots.map(lambda intrvl: (intrvl.end + 1, intrvl.end + 1, intrvl.payload))
).coalesce()
boundary_frames = {
    video_id: [
        intrvl.start
        for intrvl in shot_boundaries.get_intervallist(video_id).get_intervals()
    ]
    for video_id in shot_boundaries.get_allintervals()
}
video_ids = sorted(list(clips.get_allintervals().keys()))
frames_per_video = {
    video_id: sorted([
        f
        for interval in clips.get_intervallist(video_id).get_intervals()
        for f in range(interval.start, interval.end + 2)
    ])
    for video_id in video_ids
}
ground_truth = {
    video_id: [
        1 if f in boundary_frames[video_id] else 2
        for f in frames_per_video[video_id]
    ] 
    for video_id in video_ids
}<jupyter_output><empty_output><jupyter_text>Load Weak Labels<jupyter_code>with open('../../data/frame_counts.pkl', 'rb') as f:
    frame_counts = pickle.load(f)
labeling_function_folder = '../../data/shot_detection_weak_labels/cinematic_shots'
windows = VideoIntervalCollection({
    video_id: [
        (f, f + 16, 0)
        for f in range(0, frame_counts[video_id] - 16, 8)
    ]
    for video_id in list(frame_counts.keys())
})
preds_frames = []
preds_windows = []
for video_id in tqdm(sorted(list(frame_counts.keys()))):
    with open(os.path.join(labeling_function_folder, '{}.pkl'.format(video_id)), 'rb') as f:
        pos_boundaries, neg_boundaries = pickle.load(f)
    pos_intervallist = IntervalList([
        (b, b, 1) for b in pos_boundaries
    ])
    all_frames = IntervalList([
        (f, f, 0) for f in range(1, frame_counts[video_id] + 1)
    ])
    frames_with_labels = all_frames.set_union(pos_intervallist).coalesce(
        predicate = equal(),
        payload_merge_op = lambda p1, p2: max(p1, p2)
    )
    
    for f in frames_with_labels.get_intervals():
        preds_frames.append(((video_id, f.start), (0.0, 1.0) if f.payload == 0 else (1.0, 0.0)))
    
    window_intervallist = windows.get_intervallist(video_id)
    windows_with_boundaries = window_intervallist.join(
        pos_intervallist,
        predicate = lambda window, frame: frame.start >= window.start and frame.end < window.end,
        merge_op = lambda window, frame: [(window.start, window.end, 1)],
        working_window = 1
    )
    
    windows_with_labels = windows_with_boundaries.set_union(
        window_intervallist
    ).coalesce(
        predicate = equal(),
        payload_merge_op = lambda p1, p2: max(p1, p2)
    )
    for window in windows_with_labels.get_intervals():
        preds_windows.append(((video_id, window.start, window.end),
                              (0.0, 1.0) if window.payload == 0 or window.start == 0 else (1.0, 0.0)))
preds_np_frames = np.array(preds_frames)
with open('../../data/shot_detection_weak_labels/noisy_labels_heuristics_frame.npy', 'wb') as f:
    np.save(f, preds_np_frames)
preds_np_windows = np.array(preds_windows)
with open('../../data/shot_detection_weak_labels/noisy_labels_heuristics_windows.npy', 'wb') as f:
    np.save(f, preds_np_windows)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13, 'fraction_numerical': 0.013428571428571429, 'mean_word_length': 3.454198473282443, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '353444', 'license': 'Apache-2.0', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1249, 'n_words': 241, 'path': 'app/notebooks/metal/shot_detection_weak_labels_heuristics.ipynb', 'repo_name': 'DanFu09/esper', 'size': '3500'}