<jupyter_start><jupyter_text>Data loading & cleaning<jupyter_code>clip_res, bias_res = [], []
clip_res.append(pd.read_csv(os.path.join(DIR, "untrained_test_clip_audit_results.csv"), skipinitialspace=True).rename(columns=lambda x: str(x).strip()))
clip_res.append(pd.read_csv(os.path.join(DIR, "clip_results.csv"), skipinitialspace=True).rename(columns=lambda x: str(x).strip()))
bias_res.append(pd.read_csv(os.path.join(DIR, "untrained_test_bias_results.csv"), skipinitialspace=True).rename(columns=lambda x: str(x).strip()))
bias_res.append(pd.read_csv(os.path.join(DIR, "bias_results.csv"), skipinitialspace=True).rename(columns=lambda x: str(x).strip()))
for dfs in (clip_res, bias_res):
    for df in dfs[1:]:
        dfs[0] = dfs[0].append(df)
clip_res, bias_res = clip_res[0].drop(columns=["Unnamed: 0"]).reset_index(drop=True), bias_res[0].drop(columns=["Unnamed: 0"]).reset_index(drop=True)

import re
def name_formatted(val: str):
    val = val.strip()
    if val.startswith("best_"):
        v = val.split("neptune_run_OXVLB-")[1]
        r, e, s = tuple(re.findall(r"\d+", v))
        return f"trained_{r}"
    else: return f"untrained_{val}"
bias_res["model_name"] = bias_res["model_name"].map(name_formatted)
clip_res["model_name"] = clip_res["model_name"].map(name_formatted)
bias_res["trained"] = bias_res["model_name"].str.startswith("trained_")
clip_res["trained"] = clip_res["model_name"].str.startswith("trained_")
bias_res["long_model_name"] = bias_res["model_name"]
bias_res["model_name"] = bias_res["long_model_name"].map(lambda x: "_".join(x.split("_")[1:]))

bias_res<jupyter_output><empty_output><jupyter_text>Plotting Code<jupyter_code>aggregated_means = {}
debias_classes = ["gender", "race"]
metrics = ["ndkl", "maxskew"]
datasets = ["FairFaceVal", "UTKFaceVal", "COCOGenderVal"]

import numpy as np
filtered_bias_res = bias_res[bias_res.isin({"evaluation": metrics, "debias_class": debias_classes, "dataset": datasets})[["evaluation", "debias_class", "dataset"]].all(axis=1)].copy()
filtered_bias_res["std_perc"] = filtered_bias_res["std"]/filtered_bias_res["mean"]
col_times = filtered_bias_res.dataset.unique()
fg = sns.FacetGrid(filtered_bias_res, col="evaluation", row="debias_class", margin_titles=True, hue="dataset", height=8, aspect=1.2)
fg.map(sns.barplot, "dataset", "std_perc", order=col_times)
fg.add_legend()
plt.show()

for t in (True, False):
    t_filtered_bias_res = filtered_bias_res[filtered_bias_res["trained"] == t]
    col_times = t_filtered_bias_res.model_name.unique()
    fg = sns.FacetGrid(t_filtered_bias_res, col="evaluation", row="debias_class", margin_titles=True, hue="model_name", height=4, aspect=2)
    fg.map(sns.barplot, "model_name", "std_perc", order=col_times)
    fg.add_legend()
    plt.show()

# bias prompts
train_prompts = {}
for debias_class in debias_classes:
    _train_prompts = ["a photo of a {} person"]
    _test_prompts = ["a photo of a {} person", "a cropped photo of a {} face"]
    _paired_prompts = [('good'       , 'evil'),
                       ('smart'      , 'dumb'),
                       ('attractive' , 'unattractive'),
                       ('lawful'     , 'criminal'),
                       ('friendly'   , 'unfriendly'),
                       ('clever'     , 'stupid'),
                       ('heroic'     , 'villainous'),
                       ('hardworking', 'lazy'),
                       ('trustworthy', 'dangerous'),
                       ('caring'     , 'nasty')]
    _prompts_n_train = len(_paired_prompts)
    prompt_cfg = debias.prepare_prompt_cfg(debias_class, _paired_prompts, _train_prompts, _test_prompts, _prompts_n_train, test_on_train=False)
    train_prompts[debias_class] = {"train_prompts:": prompt_cfg["BIAS_TRAIN_PROMPTS"], "test_prompts": prompt_cfg["BIAS_TEST_PROMPTS"]}


train_templates = _train_prompts
test_templates = src.datasets.PROMPT_TEMPLATES["prompt_iterations"][src.datasets.PROMPT_TEMPLATES["prompt_iterations"]["group"]=="pairwise_adjectives"]["template"].tolist() + _test_prompts
test_templates = list(set(test_templates))
with open(os.path.join(src.PATHS.IAT.PROMPTS, "pairwise_adjectives.csv"), mode="r") as fill_file:
    fills = []
    for line in fill_file:
        fills.extend([x.strip() for x in line.split(",")])
    for pair in _paired_prompts:
        fills.extend(list(pair))
    fills = list(set(fills))
fills, train_templates, test_templates<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1637677885701378, 'fraction_numerical': 0.00338829907386492, 'mean_word_length': 4.943624161073825, 'pattern_counts': {'":': 5, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '232926', 'license': 'MIT', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1570, 'n_words': 236, 'path': 'plots/metrics_plotting.ipynb', 'repo_name': 'oxai/debias-vision-lang', 'size': '4427'}