<jupyter_start><jupyter_text>Agreement between human annotatorsMinimum IoU between re-annotated moments.<jupyter_code>def compute_agreement(x):
    st_perc = x["Answer.start"]
    ed_perc = x["Answer.end"]
    pred = np.vstack([st_perc, ed_perc]).T
    ious = squareform(pdist(pred, pair_iou))
    ious = ious[~np.eye(ious.shape[0],dtype=bool)].reshape(len(ious), -1)
    min_ious = ious.min()
    return min_ious

agreement = data.groupby("HITId").apply(compute_agreement).values
plt.figure(figsize=(10, 3))
plt.hist(agreement, bins=30)
sns.despine()
plt.title(f"Median min IoU: {np.median(agreement):.2f}")<jupyter_output><empty_output><jupyter_text>Human performance i. Randomly sample one annotator out of five5 different annotators provided moment annotation for each sample.To evaluate human performance, one annotator is randomly selected out of 5 for each sample.<jupyter_code>from multiprocessing import Pool, cpu_count, Queue
import multiprocessing as mp

def compute_iou_with_random_human(x):
    st_perc = x["Answer.start"]
    ed_perc = x["Answer.end"]
    pred = np.vstack([st_perc, ed_perc]).T
    pred = random.choice(pred)
    
    video_id = x.iloc[0]["Input.video_url"][:-4]
    description = x.iloc[0]["Input.description"]
    for gt_instance in charade_groundtruth:
        if gt_instance[0] == (video_id, description):
            st_sec, ed_sec, length = gt_instance[1]
            break
            
    ious = _tiou(pred[None,:]*length/100, (st_sec, ed_sec))
    return ious

def one_trial(data, q):
    human_ious = data.groupby("HITId").apply(compute_iou_with_random_human).values
    r_at_1 = np.mean(np.hstack(human_ious) > .5)*100
    q.put(r_at_1)
    
def eval_random_human(data, n=100):
    q = Queue()
    processes = []
    for _ in range(n):
        p = mp.Process(target=one_trial, args=(data,q))
        p.start()
        processes.append(p)

    while(processes):
        processes.pop().join()

    metrics = []
    while(not q.empty()):
        metrics.append(q.get())
    
    return metrics

%time metrics = eval_random_human(data, 100)
print(f'[Random Human] R@1 (IoU>0.5): {np.mean(metrics):.1f} ({np.std(metrics):.2f})')<jupyter_output>CPU times: user 0 ns, sys: 908 ms, total: 908 ms
Wall time: 8.77 s
[Random Human] R@1 (IoU>0.5): 42.7 (1.13)<jupyter_text>ii. Select a representative annotatorWe choose one representative annotator for each sample.To obtain one representative annotator out of 5, we compute the pairwise IoUs among the moments from the 5 annotations (consequently, each annotator get 4 IoUs) and select one annotator as representative that have the largest average IoU. Representative annotators therefore are ones who are the most consistent with others.<jupyter_code>def eval_representative_human(x):
    st_perc = x["Answer.start"]
    ed_perc = x["Answer.end"]
    pred = np.vstack([st_perc, ed_perc]).T
    ious = squareform(pdist(pred, pair_iou))
    ious = ious[~np.eye(ious.shape[0],dtype=bool)].reshape(len(ious), -1)
    index = np.argmax(np.mean(ious, axis=1))
    pred = pred[index]

    video_id = x.iloc[0]["Input.video_url"][:-4]
    description = x.iloc[0]["Input.description"]
    for gt_instance in charade_groundtruth:
        if gt_instance[0] == (video_id, description):
            st_sec, ed_sec, length = gt_instance[1]
            break
    pred = pred / 100 * length
    ious = _tiou(pred[None, :], (st_sec, ed_sec))
    return ious[0]
    
rep_human_ious = data.groupby("HITId").apply(eval_representative_human).values.ravel()
metric = np.mean(np.hstack(rep_human_ious) > .5)*100

plt.figure(figsize=(10, 3))
plt.hist(np.hstack(rep_human_ious), bins=30)
plt.title("IoUs to Charade-STA's ground truth moments")
sns.despine()
plt.show()

print(f"[Rep. Human] R@1 (IoU>0.5): {np.mean(metric):.1f}")
# output representative human response in the format for our toolbox

def convertAMTdata2result():
    
    for _, x in data.groupby("HITId"):
        st_perc = x["Answer.start"]
        ed_perc = x["Answer.end"]
        pred = np.vstack([st_perc, ed_perc]).T
        ious = squareform(pdist(pred, pair_iou))
        ious = ious[~np.eye(ious.shape[0],dtype=bool)].reshape(len(ious), -1)
        index = np.argmax(np.mean(ious, axis=1))
        pred = pred[index]

        video_id = x.iloc[0]["Input.video_url"][:-4]
        description = x.iloc[0]["Input.description"]
        for gt_instance in charade_groundtruth:
            if gt_instance[0] == (video_id, description):
                st_sec, ed_sec, length = gt_instance[1]
                break
        pred = pred / 100 * length
        iou_val = _tiou(pred[None, :], (st_sec, ed_sec))
        res = {'R@1 IoU>0.5': iou_val[0]>.5}
        yield (gt_instance[0], [pred.tolist()+[length]], [1], res)
        

def plot_per_duration():
    predictions = list(convertAMTdata2result())
    ground_truth = []
    for instance in predictions:
        for gt_instance in charade_groundtruth:
            if gt_instance[0] == instance[0]:
                ground_truth.append(gt_instance)
    plt.figure()
    plot_performance_per_duration(predictions, ground_truth, ax=plt.gca())
    plt.ylabel("# samples")
    plt.xlabel("duration")
    plt.title("Representative Human")
    sns.despine()
    
plot_per_duration()<jupyter_output><empty_output><jupyter_text>Performance of k-th Best Human<jupyter_code>def compute_iou_at_k(x, k=[0, 1, 2, 3, 4]):
    st_perc = x["Answer.start"]
    ed_perc = x["Answer.end"]
    video_id = x.iloc[0]["Input.video_url"][:-4]
    description = x.iloc[0]["Input.description"]
    for gt_instance in charade_groundtruth:
        if gt_instance[0] == (video_id, description):
            st_sec, ed_sec, length = gt_instance[1]
            break
    pred = np.vstack([st_perc, ed_perc]).T / 100 * length
    ious = _tiou(pred, (st_sec, ed_sec))
    ious = ious[np.argsort(ious)[::-1][k]]
    return ious

human_ious = data.groupby("HITId").apply(lambda x: compute_iou_at_k(x))
human_ious = np.vstack(human_ious.values)
f, axes = plt.subplots(5, 1, figsize=(10, 10), sharex=True, sharey=True)
for ax, k in zip(axes, [0,1,2,3,4]):
    human_at_k = human_ious[:, k]
    ax.hist(human_at_k, bins=30)
    ax.set_title(f"{k+1}-best human: "+f"R@1(IoU>0.5):{np.mean(human_at_k > .5)*100:.1f}")
sns.despine()<jupyter_output><empty_output><jupyter_text>Human annotation example<jupyter_code>from matplotlib.patches import ConnectionPatch
from textwrap import wrap
from skvideo.io import vread

sns.set_style("white")

def get_example(data):
    hitid = data["HITId"].sample().values[0]
    sample_data = data[data["HITId"] == hitid][["Input.video_url", "Input.description", "Answer.start", "Answer.end"]]
    video_id = sample_data["Input.video_url"].values[0]
    description = sample_data["Input.description"].values[0]
    for gt_instance in charade_groundtruth:
        if gt_instance[0] == (video_id[:-4], description):
            st_sec, ed_sec, length = gt_instance[1]
    return video_id, description, st_sec, ed_sec, length, sample_data

def show_example(data, savefig=False):
    video_id, description, st_sec, ed_sec, length, sample_data = get_example(data)
    f = plt.figure(figsize=(11*3, 3*3), constrained_layout=True)

    gs = f.add_gridspec(2, 6)

    video_path = f"./data/raw/video/{video_id}"
    video_arr = vread(video_path)

    n_frame = len(video_arr)
    sample_sec = np.linspace(0, int(length), 8).astype("i")
    sample_sec = sample_sec[1:-1]
    frames = [video_arr[int(n_frame / length * sec)] for sec in sample_sec]

    ax_timeline= f.add_subplot(gs[1, :])

    for i, frame in enumerate(frames):
        img_ax = f.add_subplot(gs[0, i])
        img_ax.imshow(frame)
        img_ax.axis("off")
        h, w, _ = frame.shape
        con = ConnectionPatch(xyA=[sample_sec[i], 1], xyB=[w*0.5, 0], coordsA="data", coordsB="axes points",
                          axesA=ax_timeline, axesB=img_ax, color='k', linestyle="--", linewidth=2)
        ax_timeline.add_artist(con)

    ax_timeline.axvspan(st_sec, ed_sec, color="#606060", alpha=0.7)
    ax_timeline.set_xlim([0, length])

    y = 0
    for _, row in sample_data.iterrows():
        ax_timeline.axvspan(row["Answer.start"] * length/100,
                    row["Answer.end"]*length/100,
                   y,y+0.2,
                   facecolor='#6002ee',
                    alpha=0.6
                   )
        y += 0.2

    for s in sample_sec:
        ax_timeline.axvline(s, linestyle="--", color="k", linewidth=2)

    ax_timeline.set_xticks(np.linspace(0, length, 10))
    ax_timeline.set_xticklabels([f'{s:.1f}s' for s in np.linspace(0, length, 10)], fontsize=24)
    ax_timeline.set_yticks(np.linspace(0.1, 0.9, 5))
    ax_timeline.set_yticklabels([f'A{a_id}' for a_id in [5,4,3,2,1]], fontsize=24)
    ax_timeline.xaxis.grid()
    sns.despine()
    plt.sca(ax_timeline)

    description = "\n".join(wrap(description))
    plt.text(0.5*length,-0.3, description, fontsize=34, ha="center", va="top", wrap=True)
    
    if savefig:
        hitid = sample_data["HITId"].values[0]
        plt.savefig(f'reports/figures/human_annotation/{hitid}.pdf',
                   bbox_inches='tight')
    
show_example(data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14629105938655645, 'fraction_numerical': 0.023819882532086143, 'mean_word_length': 3.5384995064165845, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '616028', 'license': 'MIT', 'n_tokens_mistral': 3554, 'n_tokens_neox': 3361, 'n_words': 649, 'path': 'notebooks/report/[Human performance] Charades-STA.ipynb', 'repo_name': 'mayu-ot/hidden-challenges-MR', 'size': '9194'}