<jupyter_start><jupyter_text>Variance for Preferred Number of ClustersIn this notebook, we conduct the sensitivity analysis for our `infomap` input parameter, the preferred number of clusters,as reported in the SI.In short, we investigate the question:How do the clusterings differ if we vary the preferred number of clusters?  Preparations<jupyter_code>from cdlib import evaluation, readwrite
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("darkgrid")
plt.rcParams['figure.figsize'] = (12,9)
plt.rcParams['font.size'] = 16
def make_dfs(dataset, preferred_cluster_sizes, years, config_strs):
    dfs = []
    for year in years:
        config_str = config_strs[0]
        ref_clustering = readwrite.read_community_json(
                f'../../legal-networks-data/{dataset.lower()}/11_cluster_results/{year}_0-0_1-0_-1_a-infomap_n100_m1-0_s0_c1000.json'
            )
        clusterings = [
            readwrite.read_community_json(
                f'../../legal-networks-data/{dataset.lower()}/11_cluster_results/{year}_{config_str}.json'
            )
            for config_str in config_strs
        ]
        df = pd.DataFrame(
            {
                'Preferred Clusters': p,
                'NMI': evaluation.normalized_mutual_information(ref_clustering, c).score,
                'Rand': evaluation.adjusted_rand_index(ref_clustering, c).score,
                "Year": year
            }
            for p, c in zip(preferred_cluster_sizes, clusterings)
        )
        dfs.append(df)
    return dfs
def make_boxplot(dfs, pivot_col, y_label, save_path=None):
    pd.concat(dfs).pivot(columns='Preferred Clusters', index='Year', values=pivot_col
                    ).boxplot(notch=0, sym="",color=dict(boxes='k', whiskers='k', medians='r', caps='k'))
    plt.ylabel(y_label)
    plt.xlabel('Preferred Number of Clusters')
    labels = plt.gca().get_xticklabels()
    labels[0] = 'auto'
    plt.gca().set_xticklabels(labels)
    plt.tight_layout()
    if save_path is not None:
        plt.savefig(save_path)<jupyter_output><empty_output><jupyter_text>Computing the statistics  US<jupyter_code>dataset = 'us'
preferred_cluster_sizes = list(range(0,150+1,10)) + [200]
years = range(1994,2018+1)
config_strs = [
    f'0-0_1-0_-1_a-infomap_n{preferred_cluster_size}_m1-0_s0_c1000' 
    if preferred_cluster_size 
    else '0-0_1-0_-1_a-infomap_m1-0_s0_c1000'
    for preferred_cluster_size in preferred_cluster_sizes
]
dfs = make_dfs(dataset, preferred_cluster_sizes, years, config_strs)
make_boxplot(dfs, 'NMI', 'Normalized Mutual Information', '../graphics/preferred_number_of_modules_nmi_us.pdf')
make_boxplot(dfs, 'Rand', 'Adjusted Rand Index', '../graphics/preferred_number_of_modules_rand_us.pdf')<jupyter_output><empty_output><jupyter_text>DE<jupyter_code>dataset = 'de'
preferred_cluster_sizes = list(range(0,150+1,10)) + [200]
years = [f'{year}-01-01' for year in range(1994,2018+1)]
config_strs = [
    f'0-0_1-0_-1_a-infomap_n{preferred_cluster_size}_m1-0_s0_c1000' 
    if preferred_cluster_size 
    else '0-0_1-0_-1_a-infomap_m1-0_s0_c1000'
    for preferred_cluster_size in preferred_cluster_sizes
]
dfs = make_dfs(dataset, preferred_cluster_sizes, years, config_strs)
make_boxplot(dfs, 'NMI', 'Normalized Mutual Information', '../graphics/preferred_number_of_modules_nmi_de.pdf')
make_boxplot(dfs, 'Rand', 'Adjusted Rand Index', '../graphics/preferred_number_of_modules_rand_de.pdf')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1179678530424799, 'fraction_numerical': 0.033582089552238806, 'mean_word_length': 4.036127167630058, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '402563', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1222, 'n_words': 214, 'path': 'notebooks/si_04_variance_for_preferred_number_of_clusters.ipynb', 'repo_name': 'QuantLaw/Complex-Societies-and-Growth', 'size': '3484'}