<jupyter_start><jupyter_text>Hyperparameters (learning rate) + Figures<jupyter_code>%pylab inline
import pandas as pd
import seaborn as sns #  pip3 install seaborn==0.9.0
import numpy as np
import matplotlib.pyplot as plt

res_dir = "../results/"
fig_dir = "../figures/"<jupyter_output><empty_output><jupyter_text>Load results of Kim et al. 2018[<NAME>., <NAME>., & <NAME>.  (2018). Not-so-clevr: learning same–different relations strains feedforward neural networks. Interface focus,8(4), 20180011]sort all other results according to that baseline<jupyter_code>x = range(1, 24)
problem = [20, 7, 21, 19, 1, 22, 5, 15, 16, 6, 17, 9, 13, 23, 8, 14, 4, 12, 10, 18, 3, 11, 2]

kim = np.array([0.5607734806629834, 0.5662983425414365, 0.585635359116022, 0.606353591160221, 0.6215469613259669,
0.6367403314917127, 0.6657458563535912, 0.68646408839779, 0.761049723756906, 0.8646408839779005,
0.8853591160220995, 0.8908839779005524, 0.9060773480662982, 0.9447513812154695, 0.9502762430939227,
0.9709944751381214, 1, 1, 1, 1, 1, 1, 1]) # obtained by webplot digitiser from the figure of Kim et al. 2018


def resort_to_kim(res):
    """
    Kim et al. sorted their problems according to accuracy. This function takes a list 
    with the results [[1, acc1], [2, acc2], ...] and returns the results sorted in the 
    same way as the results of Kim et al. [acc20, acc7, ...]
    """
    r2 = np.zeros(23)
    for r in res:
        index = r[0]
        value = r[1]
        r2[problem.index(index)] = value / 100  # values between 0 and 1
    return r2<jupyter_output><empty_output><jupyter_text>Some helper functions<jupyter_code>def dataframe_to_list(df, method):
    """
    this function gets a df and returns a list containing the results sorted in the order of Kim et al
    ARGS:
        method: 'val' or 'train' : extract validation or trainingsperformance
    """
    problem = df["problem"].values.tolist()
    if method == "val":
        pc = df["pc_val"].values.tolist()
    elif method == "train":
        pc = df["pc_train"].values.tolist()

    res = []
    for i in range(len(df)):
        res.append([problem[i], pc[i] * 100])
    res = resort_to_kim(res)
    return res


def get_max(res_val, res_train, L):
    """
    return the maximum of the validation performance of the different learningrates
    return the corresponding trainingperformance
    """
    a_val = np.array([res_val[L], res_val[L + 1], res_val[L + 2]])
    a_train = np.array([res_train[L], res_train[L + 1], res_train[L + 2]])

    maxindex = a_val.argmax(axis=0)
    max_val = np.zeros(23)
    max_train = np.zeros(23)
    for i in range(23):
        max_val[i] = a_val[maxindex[i]][i]
        max_train[i] = a_train[maxindex[i]][i]
    return max_val, max_train, maxindex


def trans(res_val, res_train, data):
    """
    rearrange the data
    """
    trans_val = np.zeros([23, len(data)])
    trans_train = np.zeros([23, len(data)])
    trans_index = np.zeros([23, len(data)])  # necessary to get best lr

    count = 0
    for L in data:
        maxi_val, maxi_train, maxindex = get_max(res_val, res_train, L)
        for i in range(23):
            trans_val[i][count] = maxi_val[i]
            trans_train[i][count] = maxi_train[i]
            trans_index[i][count] = maxindex[i]
        count += 1
    return trans_val, trans_train, trans_index


def load_resnet(pretrained):
    """
    load results from a .csv file
    filter lr and number of training images
    returns a list containing the results (for val and train) 
    for different lr and number of training images as arrays 
    and the corresponding labels
    """
    df_raw = pd.read_csv(res_dir + "exp_" + pretrained + ".csv")

    res_val = []
    res_train = []
    label = []
    for num_train in [28000, 1000, 100]:
        if pretrained == 'finetune':
            lrs = [0.0003, 0.0001, 0.00006]
        elif pretrained == 'scratch':
            lrs = [0.001, 0.0006, 0.0003]
        for lr in lrs:
            df = df_raw.loc[round(df_raw["lr"], 7) == lr]
            df = df.loc[df["num_trainimages"] == num_train]

            res_val.append(dataframe_to_list(df, "val"))
            res_train.append(dataframe_to_list(df, "train"))
            label.append(str(num_train) + " Images" + ", lr: " + str(lr))
    return res_val, res_train, label


def get_maxindex(res, Ls, lrs):
    """
    This function returns the index of the lr that performs best.
    It is used to pick the correct learning rates for the evaluation on the testset (code/network/svrt_test)
    """
    res_val, res_train, label = res
    res_val, res_train, transindex = trans(res_val, res_train, Ls)
    return transindex<jupyter_output><empty_output><jupyter_text>Compare accuracies on validation set to find best learning rate<jupyter_code># load dataframes containing the accuracies
res_s = load_resnet('scratch')
res_ft = load_resnet('finetune')

indices_lr_ft = get_maxindex(res_ft, [0, 3, 6], [0.0003, 0.0001, 6e-5])
indices_lr_s = get_maxindex(res_s, [0, 3, 6], [0.001, 0.0006, 0.0003])
print("These indices were copied to network_training/svrt_test.py")
print('finetune: ', indices_lr_ft)
print('scratch: ', indices_lr_s)<jupyter_output><empty_output><jupyter_text>Main figureThis shows the accuracy of the different models (Kim et al. + different variations of ResNet) sorted by same-different and spatial tasks.<jupyter_code>res_test_s = np.load(res_dir + "testset_scratch.npy")
res_test_ft = np.load(res_dir + "testset_finetune.npy")


def add_to_df(df, acc, network):
    return df.append(
        pd.DataFrame(
            {
                "Task type": ["same- \ndifferent \ntasks"] * 9
                + ["spatial \ntasks"] * 14,
                "Accuracy": acc,
                "network": [network] * 23,
            }
        )
    )


df = pd.DataFrame({"type": [], "accuracy": [], "network": []})
plot_data = np.array(
    [
        [kim, "2-6 layer DNN (Kim et al.)\n1,000,000 \nNo"],
        [res_test_ft[:, 0], "ResNet-50 \n28,000\nYes"],
        [res_test_s[:, 0], "ResNet-50 \n28,000\nNo"],
        [res_test_ft[:, 1], "ResNet-50 \n1,000\nYes"],
        [res_test_ft[:, 2], "ResNet-50 \n100\nYes"],
    ]
)
for acc, network in plot_data:
    df = add_to_df(df, acc, network)

plt.figure(figsize=[13, 3])
fig = sns.stripplot(
    x="network",
    y="Accuracy",
    hue="Task type",
    palette=["r", "b"],
    order=plot_data[:, 1],
    jitter=0.1,
    dodge=True,
    alpha=0.5,
    zorder=1,
    size=6,
    data=df,
)
plt.xlabel("")
plt.legend(loc=(-0.18, 0.27))
plt.text(-1.4, 0.426, "Network:")
plt.text(-1.4, 0.386, "# of training images:")
plt.text(-1.4, 0.346, "Pretrained on ImageNet:")
fig.spines["top"].set_visible(False)
fig.spines["right"].set_visible(False)
# plt.savefig(fig_dir + 'stripplot.pdf', format='pdf', dpi=512, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Figure in AppendixAccuracy of Resnet is plotted in the same way as by Kim et al. 2018. For each problem the accuracy for different training set sizes is shown by the dots.<jupyter_code>def plot_kim_broad():
    """
    replot results of kim et al.
    """
    fig = plt.figure(figsize=(20, 5))
    plt.xticks(x, problem, fontsize=fontsize)
    plt.yticks(fontsize=fontsize)

    plt.tick_params(
        axis="x", which="both", bottom="off", top="off",
    )
    plt.ylim(0.5, 1)

    plt.bar(x, kim, color=color, edgecolor=[1, 1, 1], width=0.9, align="center")
    plt.bar([1], [0], color=b, edgecolor=[1, 1, 1])
    plt.legend(
        ["Same-Different", "Spatial"], loc=2, fontsize=14, bbox_to_anchor=(-0.1, 1.25)
    )
    fig.axes[0].spines["top"].set_visible(False)
    fig.axes[0].spines["right"].set_visible(False)
    plt.margins(x=0.005)
    plt.xlabel("Problem Label", fontsize=fontsize)
    plt.ylabel("Accuracy", fontsize=fontsize)

    return fig


def figure_appendix_helper(r_s, r_ft, val_test_train, n1, n2, n3, name):
    fig = plot_kim_broad()
    lightgreen = (178 / 256, 229 / 256, 207 / 256)

    # scratch
    for i in range(len(r_s)):
        plt.plot(
            [i + 0.7, i + 1, i + 1.3], r_s[i], "-", color=lightgreen, markersize=markersize, clip_on=False,
        )  # line
        plt.plot(
            [i + 0.7], r_s[i][0], "o", color=lightgreen, markersize=markersize, clip_on=False,
        )  # first point
        plt.plot(
            [i + 1, i + 1.3], r_s[i][1:3], "o", color=lightgreen, markersize=5, clip_on=False,
        )  # other points

    # finetune
    for i in range(len(r_ft)):
        plt.plot(
            [i + 0.7, i + 1, i + 1.3], r_ft[i], "-", color="green", markersize=markersize, clip_on=False,
        )  # line
        plt.plot(
            [i + 0.7], r_ft[i][0], "o", color="green", markersize=markersize, clip_on=False,
        )  # first point
        plt.plot(
            [i + 1, i + 1.3], r_ft[i][1:3], "o", color="green", markersize=5, clip_on=False,
        )  # other points
    plt.title("%sperformance: %s - %s - %s" % (val_test_train, n1, n2, n3), fontsize=fontsize)
    # plt.savefig(fig_dir + 'appendix.pdf', format='pdf', dpi=512, bbox_inches='tight')


def figure_appendix(res, Ls, name, res_s, Ls_s, res_test_s, res_test_ft):
    res_val_ft, res_train_ft, label = res
    res_val_ft, res_train_ft, tmp = trans(res_val_ft, res_train_ft, Ls)

    res_val_s, res_train_s, label_s = res_s
    res_val_s, res_train_s, tmp = trans(res_val_s, res_train_s, Ls_s)
    n1 = label[Ls[0] + 1][:-12]
    n2 = label[Ls[1] + 1][:-12]
    n3 = label[Ls[2] + 1][:-12]

    figure_appendix_helper(res_test_s, res_test_ft, "test", n1, n2, n3, name)
fontsize = 18
markersize = 8
r = sns.hls_palette(8, l=0.7)[0]
b = sns.hls_palette(8, l=0.7)[5]
m = sns.hls_palette(8, l=0.7)[6]
color = [r, r, r, r, r, r, r, r, r, b, b, b, b, b, m, b, b, b, b, b, b, b, b]

figure_appendix(res_ft, [0, 12, 15], 'ft_scratch', res_s, [0, 3, 6], res_test_s, res_test_ft)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15113222806308127, 'fraction_numerical': 0.06752931661949049, 'mean_word_length': 3.1848561759729273, 'pattern_counts': {'":': 8, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '142305', 'license': 'MIT', 'n_tokens_mistral': 4228, 'n_tokens_neox': 3757, 'n_words': 921, 'path': 'SVRT/analysis/.ipynb_checkpoints/svrt_results-checkpoint.ipynb', 'repo_name': 'bethgelab/notorious_difficulty_of_comparing_human_and_machine_perception', 'size': '9896'}