<jupyter_start><jupyter_text>Load cached results<jupyter_code>cached = CachedResults(
    estimator_class=DaskRandomForestRegressor, n_splits=n_splits, cache_dir=CACHE_DIR
)
results = cached.collate_scores(train_scores=True)
cached.get_best_params()<jupyter_output><empty_output><jupyter_text>Visualise the different hyperparameter combinations<jupyter_code>hyperparams = defaultdict(list)

for param_tuples, param_results in results.items():
    for category, scores in param_results.items():
        if len(scores) == n_splits:
            hyperparams[category].append(np.mean(scores))
            hyperparams[category + "_std"].append(np.std(scores))
        else:
            print(param_tuples, category, len(scores))
            break  # Do not append anything.
    else:
        for param, param_value in param_tuples:
            hyperparams[param].append(param_value)
hyperparams = pd.DataFrame(hyperparams)
score_keys = list(param_results)
score_std_keys = [score_key + "_std" for score_key in score_keys]
param_keys = list(set(hyperparams.columns) - set(score_keys) - set(score_std_keys))
hyperparams.fillna(-1, inplace=True)
hyperparams_gap = hyperparams[hyperparams["test_score"] > 0.68].copy()
hyperparams_gap["gap"] = hyperparams_gap["train_score"] - hyperparams_gap["test_score"]
print("Nr. of params:", len(hyperparams_gap))
plt.plot(
    hyperparams_gap["test_score"], hyperparams_gap["gap"], linestyle="", marker="o"
)
plt.xlabel("test score (R2)")
plt.ylabel("R2 gap")
plt.grid(linestyle="--", alpha=0.4)
hyperparams_gap.sort_values(by="gap")
hyperparams.sort_values(by="test_score", ascending=False)[:20]
_ = hyperparams.boxplot(column=score_keys, by=["min_samples_split", "n_estimators"])
melted = pd.melt(
    hyperparams[hyperparams["test_score"] > 0.63].drop(columns=score_std_keys),
    id_vars=param_keys,
    value_vars=score_keys,
    var_name="category",
    value_name="score",
)
melted<jupyter_output><empty_output><jupyter_text>Visualise the effect of individual parameters<jupyter_code>from alepython.ale import _sci_format

for param_key in param_keys:
    fig = plt.figure(figsize=(9, 6))

    ax = sns.boxplot(x=param_key, y="score", hue="category", data=melted)
    ax.set(ylabel="R2 Score")
    ax.grid(which="both", alpha=0.4, linestyle="--")

    if param_key == "ccp_alpha":
        ax.xaxis.set_ticklabels(
            _sci_format(
                np.array(
                    list(map(lambda x: float(x.get_text()), ax.xaxis.get_ticklabels()))
                )
            )
        )
        ax.xaxis.set_tick_params(rotation=45)<jupyter_output><empty_output><jupyter_text>Standard deviations<jupyter_code>melted_std = pd.melt(
    hyperparams[hyperparams["test_score"] > 0.63].drop(columns=score_keys),
    id_vars=param_keys,
    value_vars=score_std_keys,
    var_name="category",
    value_name="score_std",
)
melted_std
from alepython.ale import _sci_format

for param_key in param_keys:
    fig = plt.figure(figsize=(9, 6))

    ax = sns.boxplot(x=param_key, y="score_std", hue="category", data=melted_std)
    ax.set(ylabel="R2 Score")
    ax.grid(which="both", alpha=0.4, linestyle="--")

    if param_key == "ccp_alpha":
        ax.xaxis.set_ticklabels(
            _sci_format(
                np.array(
                    list(map(lambda x: float(x.get_text()), ax.xaxis.get_ticklabels()))
                )
            )
        )
        ax.xaxis.set_tick_params(rotation=45)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13329486439699942, 'fraction_numerical': 0.008655510675129832, 'mean_word_length': 4.1515601783060925, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '91902', 'license': 'MIT', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1190, 'n_words': 178, 'path': 'analysis/visualise_random_search_cv.ipynb', 'repo_name': 'akuhnregnier/empirical-fire-modelling', 'size': '3466'}