<jupyter_start><jupyter_text>150 Evals<jupyter_code>best_results = {'25evals': defaultdict(list),
                '50evals': defaultdict(list),
                'final': defaultdict(list)}
best_results_test = {'25evals': defaultdict(list),
                '50evals': defaultdict(list),
                'final': defaultdict(list)}
method_map = {'nei':'BoTorch-qNEI',
              'gs': 'GS-3,6,9',
              'gs_sampling': 'GS-3,6,9-s',
              'ei': 'BO-1',
              'lcb': 'GPyOpt-LCB',
              'lcb_logregret': 'GPyOpt-LCB(log(Regret))',
              'rs':'RS-1',
              'rs3': 'Random Search x3',
              'rs5': 'Random Search x5',
              'ei3': 'GPyOpt-EI x3',
              'ei5': 'GPyOpt-EI x5',
              'nei_logregret':'BoTorch-qNEI(log(Regret))',
              'asha':'ASHA'}
evals_used = defaultdict(list)
for fname in os.listdir("results/"):
    if fname.endswith(".csv"):
        method_name = method_map[fname.split("-")[0]]
        if fname.split("-")[0] == 'nei_logregret':
            continue
        evals = fname.split("_")[-1][:-4]
        df = pd.read_csv(f"results/{fname}")
        if 'evals' in df.columns:
            evals_used[method_name].append(df['evals'].mean())
        best_results[evals][method_name].append(np.mean(df['mse']))
        best_results_test[evals][method_name].append(np.mean(df['mse_test']))
np.mean(evals_used['GS-3,6,9'])
np.mean(evals_used['GS-3,6,9-s'])
validdf = pd.concat([pd.DataFrame({'Method': [k]*len(v), 'EPE':v}) for k, v in best_results['final'].items()]).sort_values(by='Method')
testdf = pd.concat([pd.DataFrame({'Method': [k]*len(v), 'EPE':v}) for k, v in best_results_test['final'].items()]).sort_values(by='Method')
sns.set()
fig, ax = plt.subplots(ncols=2, figsize=(12, 6))

sns.boxplot(data=validdf,
            x='Method',
            y='EPE',
            ax=ax[0],
            showmeans=True,
            meanprops={"marker":"*","markerfacecolor":"white", "markeredgecolor":"black"})
ax[0].set_title("Boston GBRT: Validation Data")
ax[0].set_xlabel("")
ax[0].set_ylabel("Loss")
# ax[0].set_ylim([7, 20])

plt.sca(ax[0])
plt.xticks(rotation=90)


sns.boxplot(data=testdf,
            x='Method',
            y='EPE',
            ax=ax[1],
            showmeans=True,
            meanprops={"marker":"*","markerfacecolor":"white", "markeredgecolor":"black"})
ax[1].set_title("Boston GBRT: Test Data")
ax[1].set_xlabel("")
ax[1].set_ylabel("")
# ax[1].set_ylim([7, 20])
plt.sca(ax[1])
plt.xticks(rotation=90)


plt.tight_layout()
validdf.to_csv('../csv-results/boston-valid.csv')
testdf.to_csv('../csv-results/boston-test.csv')
validdf.groupby('Method').agg(['count', 'mean', 'median', 'var']).sort_values(by=('EPE', 'mean'), ascending=True)
testdf.groupby('Method').agg(['count', 'mean', 'median', 'var']).sort_values(by=('EPE', 'mean'), ascending=True)
meandf = pd.DataFrame({'25': df25.groupby('method').agg(['mean'])['value']['mean'],
                       '50': df50.groupby('method').agg(['mean'])['value']['mean'],
                       '100': finaldf.groupby('method').agg(['mean'])['value']['mean']})
meandf = meandf.reindex(['25', '50', '100'], axis=1)
print(meandf.to_latex(float_format=lambda x: '%.1f' % x))<jupyter_output>\begin{tabular}{lrrr}
\toprule
{} &    25 &    50 &   100 \\
method           &       &       &       \\
\midrule
ASHA             & 125.2 & 135.0 & 134.4 \\
BoTorch-qNEI     & 149.7 & 155.2 & 158.3 \\
GPyOpt-EI        & 136.5 & 143.6 & 148.5 \\
GPyOpt-EI x3     &  93.7 & 133.5 & 151.9 \\
GPyOpt-LCB       & 141.1 & 149.7 & 148.7 \\
Random Search    & 112.0 & 130.3 & 139.1 \\
Random Search x3 &  85.6 & 102.6 & 123.0 \\
Random Search x5 &  69.6 &  89.1 & 105.7 \\
\bottomrule
\end{tabular}
{'fraction_non_alphanumeric': 0.21854127705049425, 'fraction_numerical': 0.04942559444296019, 'mean_word_length': 2.8478931140801644, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '235100', 'license': 'MIT', 'n_tokens_mistral': 1537, 'n_tokens_neox': 1422, 'n_words': 206, 'path': 'distribution-of-outcomes/bostongbrt/eval.ipynb', 'repo_name': 'LarsHH/reproducible-hyperparameter-optimization-jcgs-2019', 'size': '3743'}