<jupyter_start><jupyter_text>Impact of consensus clustering on stability<jupyter_code>import pandas as pd
import matplotlib.pylab as plt
import seaborn as sns

sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (8,6)
plt.rcParams['font.size'] = 20<jupyter_output><empty_output><jupyter_text>US<jupyter_code>def make_boxplot(dataset, metric, ylabel, save_path=None):
    df = pd.read_pickle(f'../results/variance_impact_of_consensus_clustering_{dataset}.pickle')
    fig, ax = plt.subplots()
    ax.boxplot(
        [
            df.loc[metric,:].loc[i,:]['values'].tolist() 
            for i in sorted(df.loc[metric,:].index.unique())
        ], 
        notch=0,
        sym="", 
        medianprops=dict(color='r')
    )
    ax.set_xticklabels(sorted(df.loc[metric,:].index.unique()))
    plt.ylabel(ylabel)
    plt.xlabel("Runs for Consensus Clustering")
    plt.tight_layout()
    if save_path is not None:
        plt.savefig(save_path)
make_boxplot('us_reg', 'NMI', 'Normalized Mutual Information',
             '../graphics/variance_impact_of_consensus_clustering_nmi_us_reg.pdf')
make_boxplot('us_reg', 'Rand', 'Adjusted Rand Index', 
             '../graphics/variance_impact_of_consensus_clustering_rand_us_reg.pdf')<jupyter_output><empty_output><jupyter_text>DE<jupyter_code>make_boxplot('de_reg', 'NMI', 'Normalized Mutual Information',
             '../graphics/variance_impact_of_consensus_clustering_nmi_de_reg.pdf')
make_boxplot('de_reg', 'Rand', 'Adjusted Rand Index', 
             '../graphics/variance_impact_of_consensus_clustering_rand_de_reg.pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1372670807453416, 'fraction_numerical': 0.003105590062111801, 'mean_word_length': 4.424242424242424, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '667016', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 596, 'n_tokens_neox': 573, 'n_words': 76, 'path': 'notebooks/si-06-variance-impact-of-consensus-clustering.ipynb', 'repo_name': 'QuantLaw/Measuring-Law-Over-Time', 'size': '1610'}