<jupyter_start><jupyter_text>Description It generates a figure with the agreement of the final consensus partitions with the ensemble  Modules loading<jupyter_code>%load_ext autoreload
%autoreload 2
from pathlib import Path
from IPython.display import display

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import conf<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>OUTPUT_FIGURES_DIR = Path(conf.MANUSCRIPT["FIGURES_DIR"], "clustering").resolve()
display(OUTPUT_FIGURES_DIR)
OUTPUT_FIGURES_DIR.mkdir(parents=True, exist_ok=True)<jupyter_output><empty_output><jupyter_text>Load best partitions<jupyter_code>CONSENSUS_CLUSTERING_DIR = Path(
    conf.RESULTS["CLUSTERING_DIR"], "consensus_clustering"
).resolve()

display(CONSENSUS_CLUSTERING_DIR)
input_file = Path(CONSENSUS_CLUSTERING_DIR, "best_partitions_by_k.pkl").resolve()
display(input_file)
best_partitions = pd.read_pickle(input_file)
best_partitions.shape
best_partitions.head()<jupyter_output><empty_output><jupyter_text>Threshold<jupyter_code>best_parts_stats = best_partitions["ari_median"].describe()
display(best_parts_stats)
best_threshold = best_parts_stats["75%"]
best_threshold_description = "75th percentile"
display(best_threshold)<jupyter_output><empty_output><jupyter_text>Plot of selected best partitions<jupyter_code>plot_data = best_partitions.reset_index()
display(plot_data.head(5))
_tmp = best_partitions.columns[
    ~best_partitions.columns.isin(["k", "method", "partition", "selected"])
]
assert _tmp.shape[0] == 1

_measure_col = _tmp[0]
display(_measure_col)
_selected_measure = "ARI"
_selected_stat = "Median"
with sns.plotting_context("talk", font_scale=0.75), sns.axes_style(
    "whitegrid", {"grid.linestyle": "--"}
), sns.color_palette("muted"):
    current_palette = iter(sns.color_palette())

    fig, ax = plt.subplots(figsize=(18, 6))
    ax = sns.pointplot(
        data=plot_data, x="k", y=_measure_col, color=next(current_palette)
    )
    ax.axhline(
        best_threshold,
        ls="--",
        color=next(current_palette),
        label=best_threshold_description,
    )
    ax.set_ylabel(f"Agreement with ensemble\n({_selected_stat} {_selected_measure})")
    ax.set_xlabel("Number of clusters ($k$)")
    ax.set_xticklabels(ax.get_xticklabels(), rotation=45)
    plt.legend()
    plt.grid(True)
    #     plt.tight_layout()

    output_filepath = OUTPUT_FIGURES_DIR / "selected_best_partitions_by_k.svg"
    display(output_filepath)

    plt.savefig(
        output_filepath,
        #         dpi=600,
        bbox_inches="tight",
        facecolor="white",
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11749149981110692, 'fraction_numerical': 0.007555723460521345, 'mean_word_length': 6.042553191489362, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '300769', 'license': 'BSD-2-Clause-Patent', 'n_tokens_mistral': 967, 'n_tokens_neox': 927, 'n_words': 140, 'path': 'nbs/99_manuscript/clustering/00-consensus_agreement.ipynb', 'repo_name': 'greenelab/phenoplier', 'size': '2647'}