<jupyter_start><jupyter_text>Prior-Only and Action-Aware Blind baselines<jupyter_code>cd ../../
from multiprocessing import Pool, cpu_count
import matplotlib as mpl
from typing import List
import pandas as pd
import numpy as np
import spacy
import matplotlib.pyplot as plt
from src.toolbox.utils import sentence2token, _load_top_actions
from src.toolbox.eval import evaluate, accumulate_metrics
from src.toolbox.visualization import plot_performance_per_duration
from src.toolbox.data_converters import CharadesSTA2Instances
from src.toolbox.baseline import SegmentGeneratorKDE, predict
import seaborn as sns
sns.set_style("white")
train_data = CharadesSTA2Instances(pd.read_csv('data/processed/charades/charades_train.csv'))

prior_only_generator = SegmentGeneratorKDE()
prior_only_generator.fit("base", train_data)
def parse_tokens(data):
    sentences = [query[1] for query, _ in data]
    n_cpus = cpu_count()
    with Pool(n_cpus) as p:
        tokens = p.map(sentence2token, sentences, len(sentences)//n_cpus)
    return tokens

%time tokens = parse_tokens(train_data)
top_actions = _load_top_actions("charades")
action_aware_generator = SegmentGeneratorKDE()
for action in top_actions:
    indices = [i for i, query in enumerate(tokens) if action in query[0]]
    sub_train = [train_data[i] for i in indices]
    action_aware_generator.fit(action, sub_train)
action_aware_generator.fit("base", train_data)
with mpl.rc_context(rc={'font.size': 17}):
    f, axes = plt.subplots(6, 5, figsize=(18, 22))
    axes = axes.ravel()
    for i, (action, kernel) in enumerate(action_aware_generator.kernels.items()):
        if i==len(axes):
            break
        X, Y = np.mgrid[0:1:100j, 0:1:100j]
        positions = np.vstack([X.ravel(), Y.ravel()])
        Z = np.reshape(kernel(positions).T, X.shape)
        im = axes[i].imshow(np.rot90(Z), cmap=plt.cm.jet, extent=[0, 1, 0, 1], vmin=0, vmax=8)
        axes[i].text(0.9, 0.9, action, ha='right', va='top', color='w', fontsize=26)
        axes[i].grid(False)
        axes[i].set_yticks([0.2, 0.6, 1.0])
        axes[i].set_xticks([0.0, 0.5, 1.0])

        if i%5:
            axes[i].set_yticks([])
        if i<25:
            axes[i].set_xticks([])
    cbar_ax = f.add_axes([0.95, 0.1, 0.02, 0.3])#[left, bottom, width, height]
    f.colorbar(im, cax=cbar_ax)
    plt.savefig(f"reports/figures/dataset-visualization/charade-moment-prior.pdf", bbox_inches="tight")
with mpl.rc_context(rc={'font.size': 25}):
    f, axes = plt.subplots(1, 3, figsize=(20, 6))
    axes = axes.ravel()
    for i, action in enumerate(['base', 'open', 'leave']):
        kernel = action_aware_generator.kernels[action]
        if i==len(axes):
            break
        X, Y = np.mgrid[0:1:100j, 0:1:100j]
        positions = np.vstack([X.ravel(), Y.ravel()])
        Z = np.reshape(kernel(positions).T, X.shape)
        im = axes[i].imshow(np.rot90(Z), cmap=plt.cm.jet, extent=[0, 1, 0, 1], vmin=0, vmax=8)
        axes[i].text(0.9, 0.9, action, ha='right', va='top', color='w', fontsize=36)
        axes[i].grid(False)
        axes[i].set_xlabel('start')

        if i>0:
            axes[i].set_yticks([])
        else:
            axes[i].set_ylabel('duration')
            
    cbar_ax = f.add_axes([0.95, 0.15, 0.02, 0.7])#[left, bottom, width, height]
    f.colorbar(im, cax=cbar_ax)
    plt.savefig(f"reports/figures/dataset-visualization/charade-moment-prior_(base,open,leave).pdf", bbox_inches="tight")<jupyter_output><empty_output><jupyter_text>Test blind baselines<jupyter_code>test_data = CharadesSTA2Instances(pd.read_csv("data/processed/charades/charades_test.csv"))
# action-agnostic baseline
action_agnostic_predictions = predict(prior_only_generator, test_data, 0.45)
action_agnostic_results = evaluate(test_data, action_agnostic_predictions)
action_agnostic_summary = accumulate_metrics(action_agnostic_results)
f"{action_agnostic_summary['R@1 IoU>0.5'] * 100:.2f}"
def display_score(bar, color="w"):
    plt.text(
        bar.get_x()+bar.get_width()*.5,
        bar.get_height()-6,
        f"{bar.get_height():.1f}",
        horizontalalignment="center",
        fontsize=12,
        color=color
    )

from typing import Dict, Optional

def plot_performance_summary(summary: Dict[str, float], title: Optional[str] = None):
    plt.figure(figsize=(6, 4))
    c = ["#d602ee", "#df55f2", "#e98df5"]
    labels = ["Original", "Randomized"]
    keys = [f"R@{k} IoU>0.5" for k in [1, 5, 10]]
    x = np.arange(3)+0.2

    for metric in ["R@1", "R@5", "R@10"]:
        vals = [summary[k] * 100 for k in summary.keys() if k.split()[0]==metric]
        bars = plt.bar(x, vals, width=0.3, color=c.pop(0), label=metric)
        for b in bars:
            display_score(b, color="w")
        x += 0.3
        
    plt.xticks(
        ticks=np.arange(3)+0.4,
        labels=["IoU>0.3", "IoU>0.5", "IoU>0.7"],
        ha="center")
        
    plt.legend()
    if title is not None:
        plt.title(title)
    sns.despine(left=False)

plot_performance_summary(action_agnostic_summary, title="Prior-Only")
# use only first seen verb
action_aware_predictions = predict(action_aware_generator, test_data, 0.45)
action_aware_results = evaluate(test_data, action_aware_predictions)
action_aware_summary = accumulate_metrics(action_aware_results)
f"{action_aware_summary['R@1 IoU>0.5'] * 100:.2f}"
plot_performance_summary(action_aware_summary, title="Action-Aware Blind")
from src.toolbox.visualization import plot_performance_per_duration

f, axes = plt.subplots(1, 2, figsize=(20, 5))
plt.rcParams.update({'font.size': 14})
_ = plot_performance_per_duration(action_agnostic_results, test_data, ax=axes[0])
axes[0].set_title("Prior-Only")
_ = plot_performance_per_duration(action_aware_results, test_data, ax=axes[1])
axes[1].set_title("Action-Aware Blind")
plt.savefig("reports/figures/performance-analysis/charades/performance_per_duration_blindbaselines.pdf", bbox_inches="tight")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14967243406685704, 'fraction_numerical': 0.031076768016126324, 'mean_word_length': 4.080204778156997, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '413525', 'license': 'MIT', 'n_tokens_mistral': 2318, 'n_tokens_neox': 2158, 'n_words': 404, 'path': 'notebooks/report/[Blind baselines] Charades-STA.ipynb', 'repo_name': 'mayu-ot/hidden-challenges-MR', 'size': '5953'}