<jupyter_start><jupyter_text>this notebook will perform a hierarchical clustering on the relevance maps Written by <NAME> & CBIG under MIT license: https://github.com/ThomasYeoLab/CBIG/blob/master/LICENSE.md<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import scipy.io as sio
from scipy import stats
import scipy.cluster.hierarchy as sch
import os
# set random seed
np.random.seed(seed=1)
# function to perform a clustering and plot a correlation matrix and dendrogram
def cluster_mat(array, list_var, pdist_metric, link_metric, c_dict, thresh=3, path_out=None, fname=None):
    df = pd.DataFrame(data=array, columns=list_var)
    df_corr = df[list_var].corr()
    df_h = df_corr.copy()
    corr = df_h.values
    # perform the clustering
    # see sch.distance.pdist function for acceptable values for metric
    pdist = sch.distance.pdist(corr, metric=pdist_metric)
    # see sch.linkage function for acceptable values for method and metric
    linkage = sch.linkage(pdist, method=link_metric, metric=pdist_metric)
    idx = sch.fcluster(linkage, t=3, criterion='maxclust')
    
    # plot dendrogram
    plt.figure(figsize=(12,4))
    plt.ylabel('distance')
    r = sch.dendrogram(linkage, color_threshold=thresh, labels=list_var, leaf_rotation=90)
    ax = plt.gca()
    xlbls = ax.get_xmajorticklabels()
    for lbl in xlbls:
        lbl.set_color(c_dict[lbl.get_text()])
    if isinstance(path_out, str):
        plt.savefig(path_out + '/dendrogram_' + fname + '.pdf')
    
    # plot rearranged clustered correlation matrix
    cols = r['ivl']
    df_h = df_h.reindex_axis(cols,axis=1)
    df_h = df_h.reindex_axis(cols,axis=0)
    #plt.figure(figsize=(25,18.75))
    plt.figure(figsize=(10,8))
    g = sns.heatmap(df_h, vmin=-0.8, vmax=0.8, cmap="BrBG",
            xticklabels=df_h.columns,
            yticklabels=df_h.columns)
    for tick_label in g.get_yticklabels():
        tick_text = tick_label.get_text()
        tick_label.set_color(c_dict[tick_text])
    for tick_label in g.get_xticklabels():
        tick_text = tick_label.get_text()
        tick_label.set_color(c_dict[tick_text])
    if isinstance(path_out, str):
        plt.savefig(path_out + '/sim_matrix_' + fname + '.pdf')
    return idx, df_h, pdist, cols
# function to plot a correlation matrix without clustering
# size is a tuple e.g.(10,8)
def plot_raw_matrix(array, list_var, c_dict, size, path_out, fname):
    df = pd.DataFrame(data=array, columns=list_var)
    df_corr = df[list_var].corr()
    plt.figure(figsize=size)
    g = sns.heatmap(df_corr, vmin=-0.8, vmax=0.8, cmap="BrBG",
               xticklabels=df_corr.columns,
               yticklabels=df_corr.columns)
    for tick_label in g.get_yticklabels():
        tick_text = tick_label.get_text()
        tick_label.set_color(c_dict[tick_text])
    for tick_label in g.get_xticklabels():
        tick_text = tick_label.get_text()
        tick_label.set_color(c_dict[tick_text])
    if isinstance(path_out, str):
        plt.savefig(path_out + '/sim_matrix_' + fname + '.pdf')
def rearrange_matrix_sig(array, idx):
    nn = 0
    new_mat = np.zeros((array.shape[0], len(idx)))
    for i in idx:
        new_mat[:,nn] = array[:,i]
        nn += 1
    return new_mat<jupyter_output><empty_output><jupyter_text>SET OUTPUT DIRECTORY<jupyter_code># set output directory, default is None - no output will be saved unless changed
path_out = None<jupyter_output><empty_output><jupyter_text>set up variables<jupyter_code># input directory
in_dir = os.getenv('CBIG_CODE_DIR') + '/stable_projects/predict_phenotypes/ChenTam2021_TRBPC/figure_utilities/input'
# replication directory
rep_dir = os.getenv('CBIG_REPDATA_DIR') + '/stable_projects/predict_phenotypes/ChenTam2021_TRBPC/figures'
# get a list of all predicted variables/outcomes
list_var = []
with open(in_dir + '/variables_to_predict.txt') as file:
    for line in file:
        line = line.strip()
        list_var.append(line)
# list of behaviors that were significantly predicted
list_sig_r = ['vocabulary','working memory','reading',
              'crystallized cognition','attention','overall cognition',
              'executive function','fluid cognition','processing speed',
              'episodic memory','short delay recall','long delay recall',
              'fluid intelligence','visuospatial accuracy','visuospatial efficiency',
              'visuospatial reaction time','lack of planning','sensation seeking',
              'lack perseverance','negative urgency','positive urgency',
              'behavioral inhibition','fun seeking','reward responsiveness',
              'drive','total psychosis symptoms','psychosis severity',
              'somatic complaints','thought problems','attention problems',
              'anxious depressed','social problems']
# get indices from original list (list_var) that match the list of significant scores
idx_sig = [list_var.index(i) for i in list_sig_r]
# dictionary for colours of different behavioural scales
c_dict = {'vocabulary':'red',
          'attention':'red',
          'working memory':'red',
          'executive function':'red',
          'processing speed':'red',
          'episodic memory':'red',
          'reading':'red',
          'fluid cognition':'red',
          'crystallized cognition':'red',
          'overall cognition':'red',
          'short delay recall':'firebrick',
          'long delay recall':'firebrick',
          'fluid intelligence':'orangered',
          'visuospatial accuracy':'lightcoral',
          'visuospatial reaction time':'lightcoral',
          'visuospatial efficiency':'lightcoral',
          'anxious depressed':'blue',
          'withdrawn depressed':'blue',
          'somatic complaints':'blue',
          'social problems':'blue',
          'thought problems':'blue',
          'attention problems':'blue',
          'rulebreaking behavior':'blue',
          'aggressive behavior':'blue',
          'total psychosis symptoms':'cornflowerblue',
          'psychosis severity':'cornflowerblue',
          'mania severity':'dodgerblue',
          'negative urgency':'black',
          'lack of planning':'black',
          'sensation seeking':'black',
          'positive urgency':'black',
          'lack perseverance':'black',
          'behavioral inhibition':'dimgrey',
          'reward responsiveness':'dimgrey',
          'drive':'dimgrey',
          'fun seeking':'dimgrey'}<jupyter_output><empty_output><jupyter_text>cluster when all the fmri conditions are stacked together<jupyter_code># load the data
mat_stack = sio.loadmat(rep_dir + '/stacked_relevance_vectors.mat')
mat_stack['stack'].shape
# take only the significant scores and rearrange
stacked_mat = rearrange_matrix_sig(mat_stack['stack'], idx_sig)
stacked_mat.shape<jupyter_output><empty_output><jupyter_text>do clustering and plot<jupyter_code>#plt.rcParams.update({'font.size': 22})
[stacked_idx, stacked_df_h, pdist, cols]=cluster_mat(stacked_mat,
                                                    list_sig_r,
                                                    'euclidean',
                                                    'average',
                                                    c_dict,2.2,path_out,'datadriven_stacked')
# this is the order of the clustered items from above
cols<jupyter_output><empty_output><jupyter_text>correlations between behavioral categories<jupyter_code># cognition & cognition (lower triangle, no diagonal)
cog_tril = np.tril(stacked_df_h.loc['visuospatial reaction time':'fluid intelligence',
                                   'visuospatial reaction time':'fluid intelligence'].values,-1)
cog_tril[np.nonzero(cog_tril)].mean()
# cognition & personality block
stacked_df_h.loc['visuospatial reaction time':'fluid intelligence',
                 'behavioral inhibition':'drive'].values.mean()
# cognition & mental health block
stacked_df_h.loc['visuospatial reaction time':'fluid intelligence',
                 'somatic complaints':'attention problems'].values.mean()
# personality & personality (lower triangle, no diagonal)
pers_tril = np.tril(stacked_df_h.loc['behavioral inhibition':'drive',
                                   'behavioral inhibition':'drive'].values,-1)
pers_tril[np.nonzero(pers_tril)].mean()
# personality & mental health block
stacked_df_h.loc['behavioral inhibition':'drive',
                 'somatic complaints':'attention problems'].values.mean()
# mental health & mental health (lower triangle, no diagonal)
mh_tril = np.tril(stacked_df_h.loc['somatic complaints':'attention problems',
                                   'somatic complaints':'attention problems'].values,-1)
mh_tril[np.nonzero(mh_tril)].mean()<jupyter_output><empty_output><jupyter_text>giant matrix of all behavioural maps across all conditions<jupyter_code># get the indices of cols from list_var
idx_cols = [list_var.index(i) for i in cols]
# load data
mat_fc = sio.loadmat(rep_dir + '/relevance_vectors.mat')
# rearrange each fMRI matrix according to the clustering from the stacked matrix (all 4 fMRI vectors together)
rs_mat_s = rearrange_matrix_sig(mat_fc['struct_fc_vec']['rs'][0][0], idx_cols)
mid_mat_s = rearrange_matrix_sig(mat_fc['struct_fc_vec']['mid'][0][0], idx_cols)
nback_mat_s = rearrange_matrix_sig(mat_fc['struct_fc_vec']['nback'][0][0], idx_cols)
sst_mat_s = rearrange_matrix_sig(mat_fc['struct_fc_vec']['sst'][0][0], idx_cols)
# create np array with all the vectors ordered by behavior and then fmri condition
concat_mat_clus = np.zeros((87571, 128))
nn = 0
for bb in range(rs_mat_s.shape[1]): # for each sig. behavior
    concat_mat_clus[:,nn] = rs_mat_s[:,bb] # put in rs
    nn += 1
    concat_mat_clus[:,nn] = mid_mat_s[:,bb] # put in mid
    nn += 1
    concat_mat_clus[:,nn] = sst_mat_s[:,bb] # put in sst
    nn += 1
    concat_mat_clus[:,nn] = nback_mat_s[:,bb] # put in nback
    nn += 1
concat_mat_clus.shape<jupyter_output><empty_output><jupyter_text>set up dictionary for coloring labels<jupyter_code>list_fmri = ['REST ', 'MID ', 'SST ', 'NBACK ']
list_sig_r_fmri_clus = []
for behav in cols:
    for fmri in list_fmri:
        tmp = fmri + behav
        list_sig_r_fmri_clus.append(tmp)
colours = ['lightcoral', #'visuospatial reaction time',
           'black', #'lack of planning',
           'black', #'sensation seeking',
           'red', #'processing speed',
           'lightcoral', #'visuospatial efficiency',
           'red', #'episodic memory',
           'firebrick', #'short delay recall',
           'firebrick', #'long delay recall',
           'red', #'attention',
           'red', #'overall cognition',
           'red', #'executive function',
           'red', #'fluid cognition',
           'red', #'vocabulary',
           'red', #'crystallized cognition',
           'red', #'reading',
           'lightcoral', #'visuospatial accuracy',
           'red', #'working memory',
           'orangered', #'fluid intelligence',
           'dimgrey', #'behavioral inhibition',
           'black', #'negative urgency',
           'black', #'positive urgency',
           'dimgrey', #'fun seeking',
           'cornflowerblue', #'total psychosis symptoms',
           'cornflowerblue', #'psychosis severity',
           'dimgrey', #'reward responsiveness',
           'dimgrey', #'drive',
           'blue', #'somatic complaints',
           'blue', #'anxious depressed',
           'blue', #'social problems',
           'blue', #'thought problems',
           'black', #'lack perseverance',
           'blue'] #'attention problems'
colours_fmri = []
for cc in colours:
    colours_fmri.append(cc)
    colours_fmri.append(cc)
    colours_fmri.append(cc)
    colours_fmri.append(cc)
fmri_c_dict = dict(zip(list_sig_r_fmri_clus, colours_fmri))<jupyter_output><empty_output><jupyter_text>plot<jupyter_code>plt.rcParams.update({'font.size': 10})
plot_raw_matrix(concat_mat_clus, list_sig_r_fmri_clus, fmri_c_dict, (25,18.75), path_out, 'datadriven_ind')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12392770883651204, 'fraction_numerical': 0.0065794952944115935, 'mean_word_length': 3.414705882352941, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '657204', 'license': 'MIT', 'n_tokens_mistral': 4010, 'n_tokens_neox': 3708, 'n_words': 833, 'path': 'stable_projects/predict_phenotypes/ChenTam2021_TRBPC/figure_utilities/clustering/multikernel_weight_corr_clustering_datadriven.ipynb', 'repo_name': 'marielacour81/CBIG', 'size': '12011'}