<jupyter_start><jupyter_text>Measure community edge properties, and output edges > sim thresh for all communities, for Figures 2,4<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import networkx as nx
import pandas as pd
import random


# latex rendering of text in graphs
import matplotlib as mpl

from matplotlib import rcParams
rcParams['font.family'] = 'sans-serif'
rcParams['font.sans-serif'] = ['Arial']

sns.set_style('white')
sns.set_style("ticks", {"xtick.major.size": 15, "ytick.major.size": 15})
plt.rcParams['svg.fonttype'] = 'none'

import sys
from IPython.display import display

% matplotlib inline
from ddot import Ontology<jupyter_output><empty_output><jupyter_text>load cosine edge list, and gene to cluster matrix<jupyter_code>gene2term_full= pd.read_csv('new_clixo_files/gene2term_full_01_45.csv',index_col='Unnamed: 0')
display(gene2term_full.head())

sim_rank_EL = pd.read_csv('new_clixo_files/sim_rank_cosine_EL.tsv',sep='\t',names=['node1','node2','sim'])
display(sim_rank_EL.head())
len(sim_rank_EL)
# reshape sim_rank edge list into adjacency matrix (takes a few minutes)
ASD_CHD_genes = gene2term_full.index.tolist()
sim_rank = pd.DataFrame(np.zeros((len(ASD_CHD_genes),len(ASD_CHD_genes))),index=ASD_CHD_genes)
sim_rank.columns=ASD_CHD_genes

for i in sim_rank_EL.index.tolist():
    if (i%10000)==0:
        print(i) # print out some progress
    focal_row=sim_rank_EL.loc[i]
    focal_gene1=focal_row['node1']
    focal_gene2=focal_row['node2']
    focal_norm_cos_sim = focal_row['sim']
    
    sim_rank.loc[focal_gene1][focal_gene2]=focal_norm_cos_sim
    sim_rank.loc[focal_gene2][focal_gene1]=focal_norm_cos_sim # symmetric
    
sim_rank.head()<jupyter_output>0
10000
20000
30000
40000
50000
60000
70000
80000
90000
100000
110000
120000
130000
140000
150000
160000
170000
180000
190000
200000
210000
220000
230000
240000
250000
260000
270000
280000
290000
300000
310000
320000
330000
340000
350000<jupyter_text>Measure the fraction internal to external edges for a focal cluster<jupyter_code>focal_term = '901'
focal_genes = gene2term_full[gene2term_full[focal_term]==True].index.tolist()
print(len(focal_genes))
non_focal_genes = np.setdiff1d(gene2term_full.index.tolist(),focal_genes)

# pick genes randomly
focal_genes_rand = gene2term_full.index.tolist()
np.random.shuffle(focal_genes_rand)
focal_genes_rand=focal_genes_rand[0:len(focal_genes)]
non_focal_genes_rand = np.setdiff1d(gene2term_full.index.tolist(),focal_genes_rand)
sim_rank_focal = sim_rank.loc[focal_genes].T.loc[focal_genes]
sim_rank_focal_ext = sim_rank.loc[focal_genes].T.loc[non_focal_genes]

sim_rank_focal_rand = sim_rank.loc[focal_genes_rand].T.loc[focal_genes_rand]
sim_rank_focal_ext_rand = sim_rank.loc[focal_genes_rand].T.loc[non_focal_genes_rand]

# set diag to 0 (only internal, not external)
sim_rank_focal.values[[np.arange(sim_rank_focal.shape[0])]*2] = 0
sim_rank_focal_rand.values[[np.arange(sim_rank_focal_rand.shape[0])]*2] = 0
sim_rank_focal.head()
# decrease edge threshold and count number of edges

# calculate the number of interenal edges to the number external edges
thresh_list = np.linspace(.999,0,25)
num_edges_internal_list = []
num_edges_internal_rand_list = []
num_edges_external_list = []
num_edges_external_rand_list = []
for thresh in thresh_list:
    num_focal_edges = (sim_rank_focal>thresh).sum().sum()/2.0 # don't double count
    num_edges_internal_list.append(num_focal_edges)
    
    num_focal_edges = (sim_rank_focal_ext>thresh).sum().sum()
    num_edges_external_list.append(num_focal_edges)
    
    num_rand_edges = (sim_rank_focal_rand>thresh).sum().sum()/2.0 # don't double count
    num_edges_internal_rand_list.append(num_rand_edges)
    
    num_rand_edges = (sim_rank_focal_ext_rand>thresh).sum().sum()
    num_edges_external_rand_list.append(num_rand_edges)
focal_term = '901'
focal_genes = gene2term_full[gene2term_full[focal_term]==True].index.tolist()
print(len(focal_genes))
non_focal_genes = np.setdiff1d(gene2term_full.index.tolist(),focal_genes)

numreps=100
# pick genes randomly
num_edges_internal_rand=[]
num_edges_external_rand=[]
for n in np.arange(numreps):  # just loop over both rand and non-rand... even though we only need to calc non-rand once (laziness)
    if n%10==0:
        print(n)
    focal_genes_rand = gene2term_full.index.tolist()
    np.random.shuffle(focal_genes_rand)
    focal_genes_rand=focal_genes_rand[0:len(focal_genes)]
    non_focal_genes_rand = np.setdiff1d(gene2term_full.index.tolist(),focal_genes_rand)
    sim_rank_focal = sim_rank.loc[focal_genes].T.loc[focal_genes]
    sim_rank_focal_ext = sim_rank.loc[focal_genes].T.loc[non_focal_genes]

    sim_rank_focal_rand = sim_rank.loc[focal_genes_rand].T.loc[focal_genes_rand]
    sim_rank_focal_ext_rand = sim_rank.loc[focal_genes_rand].T.loc[non_focal_genes_rand]

    # set diag to 0 (only internal, not external)
    sim_rank_focal.values[[np.arange(sim_rank_focal.shape[0])]*2] = 0
    sim_rank_focal_rand.values[[np.arange(sim_rank_focal_rand.shape[0])]*2] = 0
    sim_rank_focal.head()

    # decrease edge threshold and count number of edges

    # calculate the number of interenal edges to the number external edges
    thresh_list = np.linspace(.999,0,25)
    num_edges_internal_list = []
    num_edges_internal_rand_list = []
    num_edges_external_list = []
    num_edges_external_rand_list = []
    for thresh in thresh_list:
        num_focal_edges = (sim_rank_focal>thresh).sum().sum()/2.0 # don't double count
        num_edges_internal_list.append(num_focal_edges)

        num_focal_edges = (sim_rank_focal_ext>thresh).sum().sum()
        num_edges_external_list.append(num_focal_edges)

        num_rand_edges = (sim_rank_focal_rand>thresh).sum().sum()/2.0 # don't double count
        num_edges_internal_rand_list.append(num_rand_edges)

        num_rand_edges = (sim_rank_focal_ext_rand>thresh).sum().sum()
        num_edges_external_rand_list.append(num_rand_edges)
    num_edges_internal_rand.append(num_edges_internal_rand_list)
    num_edges_external_rand.append(num_edges_external_rand_list)
    
plt.figure(figsize=(2,2))
sns.set(style="ticks", context="talk")
plt.style.use("dark_background")
plt.style.use("dark_background")
num_edges_int_rand_mean = np.mean(num_edges_internal_rand,axis=0)
num_edges_int_rand_std = np.std(num_edges_internal_rand,axis=0)

plt.plot(thresh_list,np.divide(num_edges_internal_list,num_edges_internal_list[-1]),'.-',label=focal_term)
#plt.plot(thresh_list,np.divide(num_edges_int_rand_mean,num_edges_int_rand_mean[-1]),'k.-',label='rand')
plt.errorbar(thresh_list,np.divide(num_edges_int_rand_mean,num_edges_int_rand_mean[-1]),
             np.divide(num_edges_int_rand_std,num_edges_int_rand_mean[-1]),
             fmt='.-',color='white',label='rand')
plt.xlabel('edge threshold',fontsize=10)
plt.ylabel('fraction possible internal edges',fontsize=10)
plt.legend(loc='best',fontsize=10)
plt.xticks([0,.5,1],fontsize=10)
plt.yticks([0,.2,.4,.6,.8,1.0],fontsize=10)
# plt.savefig('new_clixo_files/figures/community_edge_structure/fraction_internal_edges_'+focal_term+'.png',dpi=300,bbox_inches='tight')
# plt.savefig('../../../manuscript/figures_1911/Supplement/fraction_internal_edges_'+focal_term+'.png',dpi=300,bbox_inches='tight')
# plt.savefig('../../../manuscript/figures_1911/Supplement/fraction_internal_edges_'+focal_term+'.svg',dpi=300,bbox_inches='tight')
plt.ylim([0,1.03])
plt.figure(figsize=(2,2))

rand_int_ext_mean = np.mean(np.divide(num_edges_internal_rand,num_edges_external_rand),axis=0)
rand_int_ext_std = np.std(np.divide(num_edges_internal_rand,num_edges_external_rand),axis=0)

plt.plot(thresh_list,np.divide(num_edges_internal_list,num_edges_external_list),'.-',label=focal_term)
# plt.plot(thresh_list,np.divide(num_edges_internal_rand_list,num_edges_external_rand_list),'k.-',label='rand')
plt.errorbar(thresh_list,rand_int_ext_mean,rand_int_ext_std,
             fmt='.-',color='white',label='rand')
plt.legend(loc='best',fontsize=10)
plt.ylabel('#internal edges/#external edges',fontsize=10)
plt.xlabel('edge threshold',fontsize=10)
plt.xticks([0,.5,1],fontsize=10)
plt.yticks([0,.2,.4,.6],fontsize=10)
# plt.savefig('new_clixo_files/figures/community_edge_structure/ratio_interna_external_edges_'+focal_term+'.png',dpi=300,bbox_inches='tight')
# plt.savefig('../../../manuscript/figures_1911/Supplement/ratio_interna_external_edges_'+focal_term+'.png',dpi=300,bbox_inches='tight')
# plt.savefig('../../../manuscript/figures_1911/Supplement/ratio_interna_external_edges_'+focal_term+'.svg',dpi=300,bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>write out edgelists for root clusterThen import to cytoscape along with gene2term_full, and can create any community subgraph there<jupyter_code>root_term='963'
focal_genes = gene2term_full[gene2term_full[root_term]==True].index.tolist()
print(len(focal_genes))

focal_sim_rank = sim_rank.loc[focal_genes][focal_genes]
focal_sim_rank.head()

simthresh=0.95
# simthresh=0.95

node1list,node2list,simlist = [],[],[]
genelist = focal_sim_rank.index.tolist()
for i in np.arange(len(genelist)-1):
    for j in np.arange(i+1,len(genelist)):
        node1=genelist[i]
        node2=genelist[j]
        sim = sim_rank.loc[node1][node2]
        
        if sim>simthresh: # only save strong edges... better for viz
            node1list.append(node1)
            node2list.append(node2)
            simlist.append(sim)
        
# add self-edges so we don't leave out unconnected nodes
for n1 in genelist:
    node1list.append(n1)
    node2list.append(n1)
    simlist.append(sim)
print(len(simlist))
root_sim_rank_EL = pd.DataFrame({'node1':node1list,'node2':node2list,'sim':simlist})


root_sim_rank_EL.head()  

# root_sim_rank_EL.to_csv('new_clixo_files/root_node_edges_'+str(simthresh)+'.tsv',sep='\t',index=False)<jupyter_output>844
9480
{'fraction_non_alphanumeric': 0.12636805837049048, 'fraction_numerical': 0.042764491284961494, 'mean_word_length': 6.493545937737282, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '635648', 'license': 'MIT', 'n_tokens_mistral': 4141, 'n_tokens_neox': 3800, 'n_words': 488, 'path': 'NB7_clixo_analysis/NB7C_plot_edges_in_hierarchy_branches_200604.ipynb', 'repo_name': 'ucsd-ccbb/ASD_combined_network_analysis', 'size': '9868'}