<jupyter_start><jupyter_text>Calculate correlation matrix<jupyter_code>base_dir_df = 'C:/Users/peter/2080Ti/Jupyter/Fabian_experiment/result_correlation/'
base_dir_np = 'C:/NIH/co_occurrence_5000/'
labels = [
            'Atelectasis',
            'Cardiomegaly',
            'Effusion',
            'Infiltration',
            'Mass',
            'Nodule',
            'Pneumonia',
            'Pneumothorax',
            'Consolidation',
            'Edema',
            'Emphysema',
            'Fibrosis',
            'Pleural_Thickening',
            'Hernia']
labels = sorted(labels)
correlation_df = pd.DataFrame(index=labels, columns=labels)
correlation_df.head()
for label in labels:
    file_name = label+'_result.csv'
    dir_df = os.path.join(base_dir_df, label)
    pd_file = os.path.join(dir_df, file_name)
    
    label_name = label+'_random_label.np.npy'
    dir_np = os.path.join(base_dir_np, label)
    label_file = os.path.join(dir_np, label_name)
    
    result_df = pd.read_csv(pd_file)
    random_label = np.load(label_file)
    index_control = label
    if len(result_df) == random_label.shape[0]:
        columns = result_df.columns[1:]
        for column in columns:
            if column == 'prob_Pleural_Thickening':
                column_class = 'Pleural_Thickening'
            else:
                column_class = column.split('_')[1]
            Y = result_df[column]
            X = random_label
            correlation_df.loc[index_control][column_class] = pearsonr(X, Y)[0]
#correlation_df.values.astype(float)
correlation_df=correlation_df.convert_objects(convert_numeric=True)
#correlation_df = raw_correlation_df.round(decimals=3)
f, ax = plt.subplots(figsize=(12, 10))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
heatmap = sns.heatmap(correlation_df.T,center=0, vmin=-1.0, vmax=1.0, annot=False, cbar_kws={"ticks":[-1.0,-0.25, -0.5,
                                                                                                      -0.75, 0.0, 0.25,
                                                                                                      0.5, 0.75, 1.0]})
f.savefig('heatmap.svg', dpi=800)
sns.set(font = 'Calibri')
sns.clustermap(correlation_df.T, annot=True, center=0, vmin=-1.0, vmax=1.0,
              figsize=(12,12), row_cluster=False)
labels_cluster = [
            'Cardiomegaly',
            'Atelectasis',
            'Emphysema',
            'Pneumothorax',
            'Hernia',
            'Fibrosis',
            'Pleural_Thickening',
            'Mass',
            'Nodule',
            'Edema',
            'Infiltration',
            'Pneumonia',           
            'Consolidation',
            'Effusion'
]
labels_cluster[6] = 'Pleural Thickening'
sns.set(font_scale=1.1)
correlation_df = correlation_df.rename(columns={'Pleural_Thickening': 'Pleural Thickening'})
correlation_df = correlation_df.rename(index={'Pleural_Thickening': 'Pleural Thickening'})
correlation_df = correlation_df.reindex_axis(labels_cluster, axis=1)
correlation_df = correlation_df.round(2)
correlation_df.to_csv('cluster.csv')
clustermap = sns.clustermap(correlation_df.T, annot=True, center=0, vmin=-1.0, vmax=1.0,
              figsize=(10,10), row_cluster=False, col_cluster=True, fmt='.2f')
plt.setp(clustermap.ax_heatmap.xaxis.get_majorticklabels(), rotation=45, ha="right",
         rotation_mode="anchor")


clustermap.savefig("output.svg")
import scipy
import scipy.cluster.hierarchy as sch
from scipy.cluster.hierarchy import linkage, dendrogram
x = correlation_df.values
x.shape
d = sch.distance.pdist(x)
L = sch.linkage(d, method='average')
ind = sch.fcluster(L, 0.5*d.max())
columns = [correlation_df.columns.tolist()[i] for i in list((np.argsort(ind)))]

df = correlation_df.reindex_axis(columns, axis=1)

f, ax = plt.subplots(figsize=(8, 7))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(df, annot=False)
fig, axes = plt.subplots(figsize=(6, 3))
dend = dendrogram(linkage(d, method='complete'), 
           color_threshold=2, 
           leaf_font_size=4,
           labels = correlation_df.columns)
dend['ivl']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1307394876752054, 'fraction_numerical': 0.022232962783953602, 'mean_word_length': 2.6022628372497825, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '294003', 'license': 'MIT', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1433, 'n_words': 214, 'path': 'Correlation/Correlation_plot.ipynb', 'repo_name': 'peterhan91/Throax_GAN', 'size': '4138'}