<jupyter_start><jupyter_text>Description This notebooks checks which traits move from different partitions and clusters across the clustering tree.  Modules loading<jupyter_code>%load_ext autoreload
%autoreload 2
from IPython.display import display
from pathlib import Path

import pandas as pd

from data.cache import read_data
from utils import generate_result_set_name
import conf<jupyter_output><empty_output><jupyter_text>Settings  Load data  S-MultiXcan projection (`z_score_std`)<jupyter_code>INPUT_SUBSET = "z_score_std"
INPUT_STEM = "projection-smultixcan-efo_partial-mashr-zscores"
input_filepath = Path(
    conf.RESULTS["DATA_TRANSFORMATIONS_DIR"],
    INPUT_SUBSET,
    f"{INPUT_SUBSET}-{INPUT_STEM}.pkl",
).resolve()
display(input_filepath)

assert input_filepath.exists(), "Input file does not exist"

input_filepath_stem = input_filepath.stem
display(input_filepath_stem)
data = read_data(input_filepath)
data.shape
data.head()<jupyter_output><empty_output><jupyter_text>Clustering results<jupyter_code>CONSENSUS_CLUSTERING_DIR = Path(
    conf.RESULTS["CLUSTERING_DIR"], "consensus_clustering"
).resolve()

display(CONSENSUS_CLUSTERING_DIR)
input_file = Path(CONSENSUS_CLUSTERING_DIR, "best_partitions_by_k.pkl").resolve()
display(input_file)
best_partitions = read_data(input_file)
# keep selected partitions only
best_partitions = best_partitions[best_partitions["selected"]]
best_partitions.shape
best_partitions.head()<jupyter_output><empty_output><jupyter_text>Within the "complex" branch  part15 k14 to part16 k14<jupyter_code>part_one = best_partitions.loc[15, "partition"]
part_two = best_partitions.loc[16, "partition"]
part_one_clus = data.loc[part_one == 14].index
part_two_clus = data.loc[part_two == 14].index
part_one_clus
part_two_clus
part_one_clus.intersection(part_two_clus)<jupyter_output><empty_output><jupyter_text>part25 k24 to part26 k15<jupyter_code>part_one = best_partitions.loc[25, "partition"]
part_two = best_partitions.loc[26, "partition"]
part_one_clus = data.loc[part_one == 24].index
part_two_clus = data.loc[part_two == 15].index
part_one_clus
part_two_clus
part_one_clus.intersection(part_two_clus)<jupyter_output><empty_output><jupyter_text>part26 k13 to part29 k21<jupyter_code>part_one = best_partitions.loc[26, "partition"]
part_two = best_partitions.loc[29, "partition"]
part_one_clus = data.loc[part_one == 13].index
part_two_clus = data.loc[part_two == 21].index
part_one_clus
part_two_clus
part_one_clus.intersection(part_two_clus)<jupyter_output><empty_output><jupyter_text>Between branches  part22 k13 to part25 k23<jupyter_code>part_one = best_partitions.loc[22, "partition"]
part_two = best_partitions.loc[25, "partition"]
part_one_clus = data.loc[part_one == 13].index
part_two_clus = data.loc[part_two == 23].index
part_one_clus
part_two_clus
part_one_clus.intersection(part_two_clus)<jupyter_output><empty_output><jupyter_text>part26 k0 to part29 k21<jupyter_code>part_one = best_partitions.loc[26, "partition"]
part_two = best_partitions.loc[29, "partition"]
part_one_clus = data.loc[part_one == 0].index
part_two_clus = data.loc[part_two == 21].index
part_one_clus
part_two_clus
part_one_clus.intersection(part_two_clus)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1015843429636533, 'fraction_numerical': 0.024541783162472817, 'mean_word_length': 9.353697749196142, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '655798', 'license': 'BSD-2-Clause-Patent', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1173, 'n_words': 188, 'path': 'nbs/99_manuscript/clustering/13-clustering_claims.ipynb', 'repo_name': 'greenelab/phenoplier', 'size': '3219'}