<jupyter_start><jupyter_text>Correlation in Scores Let's see how correlated the influences are before and after augmentation<jupyter_code>scipy.stats.spearmanr(initial_aug_scores, after_abs_aug_scores_cond)
small_scores_idxs = initial_aug_scores < initial_aug_scores.mean()
scipy.stats.spearmanr(initial_aug_scores[small_scores_idxs],
                      after_abs_aug_scores_cond[small_scores_idxs])
initial_aug_scores.describe()
after_abs_aug_scores.describe()
initial_aug_scores[small_scores_idxs].describe()
after_abs_aug_scores_cond[small_scores_idxs].describe()
sns.jointplot(x=initial_aug_scores[small_scores_idxs],
              y=after_abs_aug_scores_cond[small_scores_idxs],
              marginal_kws=dict(bins=10,
                                rug=False,
                                hist=False,
                                kde=True,
                                kde_kws=dict(bw=.0005))
             );<jupyter_output><empty_output><jupyter_text>Support Vectors<jupyter_code>is_SV = runs_data["is_SV"].astype(np.int)
print("There are {} support vectors".format(np.sum(is_SV)))
hist = sns.distplot(is_SV, kde=False, bins=hist_bins)
hist.set_yscale('log')
hist.get_figure().savefig("is_SV_histogram.pdf")
VSV_acc = runs_data["VSV_acc"]
VSV_acc<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>runs_data["run_parameters"]
baseline_acc = runs_data["no_aug_no_poison_acc"]
poisoned_acc = runs_data["poisoned_acc"]
all_aug_train_poisoned_acc = runs_data["all_aug_train_poisoned_acc"]
n_aug_sample_points = runs_data["n_aug_sample_points"]
n_train = runs_data["n_train"]
baseline_acc
poisoned_acc
all_aug_train_poisoned_acc
n_train
runs_data["experiment_results"].item()
labels = list(runs_data["experiment_results"].item().keys())
run_matrix = np.array([
    np.array(runs_data["experiment_results"].item()[k]) for k in labels  
])
run_matrix
run_df_rows = []
for i, label in enumerate(labels):
    for test in range(run_matrix[i].shape[0]):
        run_df_row = pd.Series()
        run_df_row["test_i"] = test
        run_df_row["n_auged"] = 0
        run_df_row["test_type"] = label
        run_df_row["test_accuracy"] = float(poisoned_acc)
        run_df_rows.append(run_df_row)
        for step in range(run_matrix[i].shape[1]):
            run_df_row = pd.Series()
            run_df_row["test_i"] = test
            run_df_row["n_auged"] = n_aug_sample_points[step]
            run_df_row["test_type"] = label
            run_df_row["test_accuracy"] = run_matrix[i][test, step]
            run_df_rows.append(run_df_row)
run_df = pd.DataFrame(run_df_rows)
run_df
#old_filename_prefix = filename_prefix + "_old"
#old_run_df = pd.read_pickle("{}.pkl".format(old_filename_prefix))
#old_run_df
#to_remove = [i for i, x in enumerate(old_run_df["test_type"]) if "update" in x]
#to_remove
#old_run_df.drop(index=to_remove,
#                inplace=True)
#run_df
#to_not_remove = [i for i, x in enumerate(run_df["test_type"]) if "update" not in x]
#run_df.drop(index=to_not_remove,
#            inplace=True)
#run_df = pd.concat([old_run_df, run_df])
run_df
run_df.to_pickle(filename_prefix + ".pkl")
run_df.to_csv(filename_prefix + ".csv", encoding='utf-8')
n_aug_sample_points = run_df["n_auged"].unique()
n_aug_sample_points
all_samples_points = np.unique(np.concatenate([[0], n_aug_sample_points]))
all_samples_points
aucs = (run_df
        .sort_values("n_auged", ascending=True)
        .groupby(["test_type", "test_i"])["test_accuracy"]
        .apply(
            lambda x: sklearn.metrics.auc(all_samples_points, x)
        )
       )
aucs
aucs.groupby("test_type").mean()
aucs.groupby("test_type").var()
auc_means = aucs.groupby("test_type").mean().sort_values(ascending=False).rename("AUC Mean")
auc_std = aucs.groupby("test_type").std().sort_values(ascending=False).rename("AUC Std.")
auc_mean_std = pd.concat([auc_means, auc_std], axis=1).sort_values(ascending=False, by="AUC Mean")
#auc_mean_std.index.name = "Test Type"
auc_mean_std
replace_dict = {x: string.capwords(x.replace("_", " ")) for x in auc_mean_std.index.unique()}
replace_dict
formatted_auc_mean_std = auc_mean_std.copy(deep=True)
formatted_auc_mean_std = formatted_auc_mean_std.replace(np.nan, "{\textemdash}")
formatted_auc_mean_std.index.name = "Policy"
formatted_auc_mean_std = formatted_auc_mean_std.reset_index()
allowed_columns = {"baseline",
                   "random_proportional",
                   "random_proportional_update",
                   "random_proportional_downweight",
                   "random_proportional_update_downweight",
                   "random_inverse_proportional",
                   "deterministic_proportional",
                   "deterministic_proportional_update",
                   "deterministic_proportional_downweight",
                   "deterministic_proportional_update_downweight",
                   "deterministic_inverse_proportional"
                  }
removed_idxs = [i for i, x in
                enumerate(formatted_auc_mean_std["Policy"])
                if x not in allowed_columns]
removed_idxs
formatted_auc_mean_std.drop(index=removed_idxs, inplace=True)
formatted_auc_mean_std["Policy"].unique()
#formatted_auc_mean_std["Policy"] = formatted_auc_mean_std["Policy"].str.replace(r"_", r"\_")
formatted_auc_mean_std["Policy"] = formatted_auc_mean_std["Policy"].replace(replace_dict, regex=False)
formatted_auc_mean_std
header_names=["{{{}}}".format(c) for c in formatted_auc_mean_std.columns]
with open(filename_prefix + '_auc_mean_std.tex', 'w') as f:
    f.write(formatted_auc_mean_std.to_latex(
        na_rep="---",
        escape=False,
        header=header_names,
        index=False,
        column_format="l*{{{right_cols}}}{{S}}".format(right_cols=len(formatted_auc_mean_std.columns)-1)
    ))
best_at_n = run_df.groupby("n_auged", as_index=False).max()
best_at_n
baseline_perf = run_df.query("test_type == 'baseline'").groupby("n_auged", as_index=False).mean()
baseline_perf
run_df.dtypes
run_plot = sns.lineplot(x="n_auged",
                        y="test_accuracy",
                        ci=95,
                        data=run_df.query("test_type == 'baseline'"))
run_plot.axhline(y=baseline_acc,
                 color="b",
                 linestyle="--",
                 label="baseline_acc")
run_plot.axhline(y=poisoned_acc,
                 color="r",
                 linestyle="--",
                 label="poisoned_acc")
run_plot.axhline(y=all_aug_train_poisoned_acc,
                 color="g",
                 linestyle="--",
                 label="all_aug_train_poisoned_acc")
run_plot = sns.lineplot(x="n_auged",
                        y="test_accuracy",
                        ci=95,
                        data=best_at_n,
                        ax=run_plot)
"""
run_plot = sns.relplot(x="n_auged",
                       y="test_accuracy",
                       hue="test_i",
                       col="test_type",
                       col_wrap=4,
                       ci=0,
                       markers=True,
                       kind="line",
                       data=run_df,
                       ax=run_plot,
                       alpha=0.1)
                       """
run_plot = sns.lineplot(x="n_auged",
                        y="test_accuracy",
                        hue="test_type",
                        ci=95,
                        data=run_df)
run_plot.axhline(y=baseline_acc,
                 color="b",
                 linestyle="--",
                 label="baseline_acc")
run_plot.axhline(y=poisoned_acc,
                 color="r",
                 linestyle="--",
                 label="poisoned_acc")
run_plot.axhline(y=all_aug_train_poisoned_acc,
                 color="g",
                 linestyle="--",
                 label="all_aug_train_poisoned_acc")
#run_plot.get_figure().savefig("test_accuracy_summary.pdf")
run_plot = sns.relplot(x="n_auged",
                       y="test_accuracy",
                       hue="test_i",
                       col="test_type",
                       col_wrap=4,
                       ci=95,
                       markers=True,
                       kind="line",
                       data=run_df)
run_plot = sns.relplot(x="n_auged",
                       y="test_accuracy",
                       hue="test_type",
                       col="test_type",
                       col_wrap=4,
                       ci=95,
                       markers=True,
                       kind="line",
                       palette=sns.color_palette("Set2", n_colors=len(run_df["test_type"].unique())),
                       data=run_df.query("n_auged <= 10"))
sns.set_style("whitegrid")
sns.set_context("paper")
sns.set(font_scale=1.1)
sns.despine(left=True)
sns.set_style("ticks", {"xtick.major.size": 8, "ytick.major.size": 8})
cmap = sns.color_palette("Set1")
sns.palplot(cmap)
sns.set_palette(cmap)
cmap = sns.color_palette("Set1")
yellow = cmap[5]
cmap[5] = cmap[6]
cmap[6] = yellow
sns.palplot(cmap)
sns.set_palette(cmap)
update_downweight_run_df = (run_df.query("test_type == 'random_proportional'"
                                          "| test_type == 'random_inverse_proportional'"
                                          "| test_type == 'baseline'")
                            .query("n_auged < 1100"))
update_downweight_run_df = update_downweight_run_df.rename(
    index=str,
    columns={
            "test_accuracy": "Test Accuracy",
            "n_auged": "Number of Augmented Points",

            },
)
update_downweight_run_df["test_type"] = update_downweight_run_df["test_type"].replace(replace_dict, regex=False)
fig, ax = plt.subplots()
run_plot = sns.lineplot(x="Number of Augmented Points",
                        y="Test Accuracy",
                        hue="test_type",
                        style="test_type",
                        ci=95,
                        data=update_downweight_run_df,
                        markers=True,
                        dashes=True,
                        ax=ax)
l = ax.legend()
#l.texts[0].set_text("")
#l.set_title('Whatever you want')
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles=handles[1:], labels=labels[1:])
ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))
plt.setp(ax.get_legend().get_texts(), fontsize='11.5') # for legend text 
#run_plot.axhline(y=baseline_acc,
#                 color="b",
#                 linestyle="--",
#                 label="baseline_acc")
run_plot.axhline(y=poisoned_acc,
                 color="r",
                 linestyle="--",
                 label="poisoned_acc")
run_plot.axhline(y=all_aug_train_poisoned_acc,
                 color="g",
                 linestyle="--",
                 label="all_aug_train_poisoned_acc")
run_plot.get_figure().savefig(filename_prefix + "_modifications_accuracy.pdf",
                              bbox_inches="tight")
update_downweight_run_df = (run_df.query("test_type == 'deterministic_proportional'"
                                          "| test_type == 'deterministic_proportional_update'"
                                          "| test_type == 'deterministic_proportional_downweight'"
                                          "| test_type == 'deterministic_proportional_update_downweight'"
                                          "| test_type == 'deterministic_inverse_proportional'"
                                          "| test_type == 'baseline'")
                            .query("n_auged < 1100"))
update_downweight_run_df = update_downweight_run_df.rename(
    index=str,
    columns={"test_accuracy": "Test Accuracy",
             "n_auged": "Number of Augmented Points",
            },
)
update_downweight_run_df["test_type"] = update_downweight_run_df["test_type"].replace(replace_dict, regex=False)
fig, ax = plt.subplots()
run_plot = sns.lineplot(x="Number of Augmented Points",
                        y="Test Accuracy",
                        hue="test_type",
                        style="test_type",
                        ci=95,
                        data=update_downweight_run_df,
                        markers=True,
                        dashes=True,
                        ax=ax)
l = ax.legend()
#l.texts[0].set_text("")
#l.set_title('Whatever you want')
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles=handles[1:], labels=labels[1:])
ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))
plt.setp(ax.get_legend().get_texts(), fontsize='10') # for legend text 
#run_plot.axhline(y=baseline_acc,
#                 color="b",
#                 linestyle="--",
#                 label="baseline_acc")
run_plot.axhline(y=poisoned_acc,
                 color="r",
                 linestyle="--",
                 label="poisoned_acc")
run_plot.axhline(y=all_aug_train_poisoned_acc,
                 color="g",
                 linestyle="--",
                 label="all_aug_train_poisoned_acc")
run_plot.get_figure().savefig(filename_prefix + "_deterministic_accuracy.pdf",
                              bbox_inches="tight")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12243969048702777, 'fraction_numerical': 0.004930966469428008, 'mean_word_length': 2.342545638945233, 'pattern_counts': {'":': 6, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '280685', 'license': 'MIT', 'n_tokens_mistral': 4415, 'n_tokens_neox': 4212, 'n_words': 524, 'path': 'Visualize_LOO_Experiments-DPP.ipynb', 'repo_name': 'mkuchnik/Efficient_Augmentation', 'size': '13182'}