<jupyter_start><jupyter_text>08/20/2020SCC Vocab Finder<jupyter_code># basic packages
import os, glob
import pandas as pd
import numpy as np; np.random.seed(0)
import itertools
from collections import Counter, defaultdict
import time
# Import tools needed for visualization
import seaborn as sns; sns.set()
import matplotlib
import matplotlib.pyplot as plt
save_dir = '../data/processed/fig4_modelling/vocab_sum/'
if not os.path.exists(save_dir):
    os.makedirs(save_dir)
TISSUE = 'SCC'
MAX_DIST=135

#get tfs
tf_df = pd.read_csv('../data/external/transcription_factor_info_063020.csv')
TFS = sorted(set(tf_df[tf_df['cell_type']=='Keratinocytes']["tf"]))##### DIFFERENT FOR EACH TISSUE
print(len(TFS))
print(TFS)

vocab_dir = '../data/processed/fig4_modelling/tf_tf_pairs/'<jupyter_output><empty_output><jupyter_text>read in loop_types<jupyter_code>glob.glob('../data/processed/fig4_modelling/tf_tf_pairs_scc/'+'*loop_type*')<jupyter_output><empty_output><jupyter_text>1 understand the types of vocabularies pulled out and where they come from 1A. enriched vocabularies from unique tissue genes<jupyter_code>loop_type_df_1 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/diff_A431-CTRLi_loop_type.csv',index_col=0).fillna('')
loop_type_df_2 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/diff_CAL27-CTRLi_loop_type.csv',index_col=0).fillna('')
loop_type_df_3 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/diff_SCC13-CTRLi_loop_type.csv',index_col=0).fillna('')

loop_type_df_all = loop_type_df_1[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]+loop_type_df_2[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]+loop_type_df_3[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]
loop_type_df_all['pro_loop_count'] = loop_type_df_all['pro_loop_count']+ loop_type_df_all['loop_pro_count']
loop_type_df_all = loop_type_df_all[['pro_pro_count','pro_loop_count','loop_loop_count']]
loop_type_df_all[:5]
# for motifs
# 1. get motifs
motifs = set()
for vocab_pair in loop_type_df_all.index.values:
    v1, v2 = vocab_pair.split('::')
    motifs.add(v1)
    motifs.add(v2)
motifs = sorted(motifs)
print(len(motifs))

# 2. initialize counts
motif_counts_dict = {}
for motif in motifs:
    motif_counts_dict[motif] = {'pro_pro_count':0,'loop_loop_count':0,'pro_loop_count':0} #list of counts for pro_pro_count, loop_loop_count, pro_loop_count, respectively
    
# 3. run through loop_type_df to fill in motif_counts_dict
for vocab_pair, row in loop_type_df_all.iterrows():
    v1, v2 = vocab_pair.split('::')
    motif_counts_dict[v1]['pro_pro_count'] +=row['pro_pro_count']
    motif_counts_dict[v2]['pro_pro_count'] +=row['pro_pro_count']
    motif_counts_dict[v1]['loop_loop_count'] +=row['loop_loop_count']
    motif_counts_dict[v2]['loop_loop_count'] +=row['loop_loop_count']
    motif_counts_dict[v1]['pro_loop_count'] +=row['pro_loop_count']
    motif_counts_dict[v2]['pro_loop_count'] +=row['pro_loop_count']
            
motif_counts_df = pd.DataFrame.from_dict(motif_counts_dict,orient='index')   
motif_counts_df_norm = motif_counts_df/motif_counts_df.sum(axis=0)
motif_counts_df_norm_sel = motif_counts_df_norm.reset_index()[motif_counts_df_norm.reset_index()['index'].isin(TFS)]
motif_counts_df_norm_sel.set_index('index',inplace=True)
# all tfs
ax = motif_counts_df_norm.sort_values('pro_pro_count').plot(kind='bar', stacked=True,figsize = (20,7),fontsize=9)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_diff_config_count_bar.pdf'))

#same graph as before but only for tissue specific TFS

# ax = motif_counts_df_norm_sel.sort_values('pro_pro_count').plot(kind='bar', stacked=True,figsize = (10,7),fontsize=12)
# ax.figure.savefig(os.path.join(save_dir, TISSUE+'_diff_config_count_sel_bar.pdf'))<jupyter_output><empty_output><jupyter_text>this shows the likelihood of the motif appears in a vocab pair from pro_pro, pro_loop, or loop_loop configurations, GIVEN that it does appear at all (so normalized by both the probability of the configuration type as well as by the motif frequency itself<jupyter_code>ax = motif_counts_df_norm.div(motif_counts_df_norm.sum(axis=1),axis=0).sort_values('pro_pro_count').plot(kind='bar', stacked=True,
                                                                                                    figsize = (20,7),fontsize=9)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_diff_config_count_norm_bar.pdf'))

# ax = motif_counts_df_norm_sel.div(motif_counts_df_norm_sel.sum(axis=1),axis=0).sort_values('pro_pro_count').plot(kind='bar', stacked=True,
#                                                                                                     figsize = (10,7),fontsize=12)
# ax.figure.savefig(os.path.join(save_dir, TISSUE+'_diff_config_count_norm_sel_bar.pdf'))<jupyter_output><empty_output><jupyter_text>1b enriched vocab from expressed genes in all scc lines<jupyter_code>loop_type_df_1 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/expr_A431-CTRLi_loop_type.csv',index_col=0).fillna('')
loop_type_df_2 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/expr_CAL27-CTRLi_loop_type.csv',index_col=0).fillna('')
loop_type_df_3 = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs_scc/expr_SCC13-CTRLi_loop_type.csv',index_col=0).fillna('')

loop_type_df_all = loop_type_df_1[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]+loop_type_df_2[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]+loop_type_df_3[['pro_pro_count','pro_loop_count','loop_pro_count','loop_loop_count']]
loop_type_df_all['pro_loop_count'] = loop_type_df_all['pro_loop_count']+ loop_type_df_all['loop_pro_count']
loop_type_df_all = loop_type_df_all[['pro_pro_count','pro_loop_count','loop_loop_count']]
loop_type_df_all[:5]
# for motifs
# 1. get motifs
motifs = set()
for vocab_pair in loop_type_df_all.index.values:
    v1, v2 = vocab_pair.split('::')
    motifs.add(v1)
    motifs.add(v2)
motifs = sorted(motifs)
print(len(motifs))

# 2. initialize counts
motif_counts_dict = {}
for motif in motifs:
    motif_counts_dict[motif] = {'pro_pro_count':0,'loop_loop_count':0,'pro_loop_count':0} #list of counts for pro_pro_count, loop_loop_count, pro_loop_count, respectively
    
# 3. run through loop_type_df to fill in motif_counts_dict
for vocab_pair, row in loop_type_df_all.iterrows():
    v1, v2 = vocab_pair.split('::')
    motif_counts_dict[v1]['pro_pro_count'] +=row['pro_pro_count']
    motif_counts_dict[v2]['pro_pro_count'] +=row['pro_pro_count']
    motif_counts_dict[v1]['loop_loop_count'] +=row['loop_loop_count']
    motif_counts_dict[v2]['loop_loop_count'] +=row['loop_loop_count']
    motif_counts_dict[v1]['pro_loop_count'] +=row['pro_loop_count']
    motif_counts_dict[v2]['pro_loop_count'] +=row['pro_loop_count']
            
motif_counts_df = pd.DataFrame.from_dict(motif_counts_dict,orient='index')   
motif_counts_df_norm = motif_counts_df/motif_counts_df.sum(axis=0)
motif_counts_df_norm_sel = motif_counts_df_norm.reset_index()[motif_counts_df_norm.reset_index()['index'].isin(TFS)]
print(motif_counts_df_norm_sel.shape)
motif_counts_df_norm_sel.set_index('index',inplace=True)
# all tfs
ax = motif_counts_df_norm.sort_values('pro_pro_count').plot(kind='bar', stacked=True,figsize = (20,7),fontsize=9)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_expr_config_count_bar.pdf'))

#same graph as before but only for tissue specific TFS

ax = motif_counts_df_norm_sel.sort_values('pro_pro_count').plot(kind='bar', stacked=True,figsize = (10,7),fontsize=12)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_expr_config_count_sel_bar.pdf'))<jupyter_output><empty_output><jupyter_text>this shows the likelihood of the motif appears in a vocab pair from pro_pro, pro_loop, or loop_loop configurations, GIVEN that it does appear at all (so normalized by both the probability of the configuration type as well as by the motif frequency itself<jupyter_code>ax = motif_counts_df_norm.div(motif_counts_df_norm.sum(axis=1),axis=0).sort_values('pro_pro_count').plot(kind='bar', stacked=True,
                                                                                                    figsize = (20,7),fontsize=9)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_expr_config_count_norm_bar.pdf'))

ax = motif_counts_df_norm_sel.div(motif_counts_df_norm_sel.sum(axis=1),axis=0).sort_values('pro_pro_count').plot(kind='bar', stacked=True,
                                                                                                    figsize = (10,7),fontsize=12)
ax.figure.savefig(os.path.join(save_dir, TISSUE+'_expr_config_count_norm_sel_bar.pdf'))<jupyter_output><empty_output><jupyter_text>2. Vocabulary genomic instances - setup 2A. find the locations<jupyter_code>sorted(glob.glob('../data/processed/fig4_modelling/tf_tf_pairs_scc/'+'*vocab*'))<jupyter_output><empty_output><jupyter_text>2B helper functions<jupyter_code>def get_other_vocab_word(row, next_row):
    if row['vocab']!=next_row['vocab']:
        return False
    
    vocab_word = set(row['tf'])
    vocab_set = set(row['vocab'].split('::'))
    other_vocab = list(vocab_set -  vocab_word)[0]
    return other_vocab == next_row['tf']

def check_distance(row,next_row,max_dist=MAX_DIST):
    if row['chr_m']==next_row['chr_m']:
        if row['stop_m']<next_row['start_m']:
            tot_dist = next_row['stop_m'] - row['start_m']
            btn_dist = next_row['start_m'] - row['stop_m']
            return (tot_dist < max_dist), tot_dist, btn_dist
    return False,-1,-1

def check_tissue(row,next_row,tfs=TFS):
    if (row['tf'] in tfs) & (next_row['tf'] in tfs):
        return 'both'
    elif (row['tf'] in tfs) | (next_row['tf'] in tfs):
        return 'one'    
    else:
        return 'none'
    
def get_hits(vocab_file,tfs=TFS):
    print('**** reading', vocab_file)
    vocab_df = pd.read_csv(vocab_file)
    print(vocab_df.shape)
    idx = 0
    idx_hits = 0
    results_dict = {}
    while idx < (vocab_df.shape[0]-1):
        # look at next 
        row = vocab_df.iloc[idx,:]
        next_row = vocab_df.iloc[idx+1,:]

        check_vocab_pair = get_other_vocab_word(row,next_row)
        check_dist,tot_dist, btn_dist = check_distance(row, next_row)
        check_tissue_tf = check_tissue(row,next_row,tfs)
        if (check_dist and check_vocab_pair):
    #         print('hi',idx)
    #         print(row)
    #         print(next_row)

            results_dict[idx_hits] = {'vocab_pair':row['vocab'],'tot_dist':tot_dist,'btn_dist':btn_dist,
                                      'chr':row['chr'],'start':row['start_m'],'stop':next_row['stop_m'], 
                                      'vocab1':row['tf'],'vocab1_start':row['start_m'], 'vocab1_stop': row['stop_m'],
                                      'vocab2':next_row['tf'],'vocab2_start':next_row['start_m'], 'vocab2_stop': next_row['stop_m'],
                                      'genes':row['genes'],'num_genes':len(row['genes'].split('|')), 'tissue':row['tissue'],
                                     'check_tissuetf':check_tissue_tf}
            idx_hits+=1

        idx+=1
    print('num_hits',idx_hits)
    results_df = pd.DataFrame.from_dict(results_dict, orient='index')
    return results_df
    
def filter_results(results_df,min_hits_per_vocab=10):
    print('shape', results_df.shape)
    vocab_counts = results_df.vocab_pair.value_counts()
    print('original num vocab',vocab_counts.shape[0])
    vocab_to_include = vocab_counts[vocab_counts>min_hits_per_vocab].index.values
    print('filt num vocab',vocab_to_include.shape[0])
    
    results_df_filt = results_df[results_df.vocab_pair.isin(vocab_to_include)]
    return results_df_filt<jupyter_output><empty_output><jupyter_text>3. Vocabulary genomic instances - running 3A get expression and stability for loop-loop regionsrun through all 3 tissue typesand then<jupyter_code># ###TESTING PURPOSES
# vocab_df = pd.read_csv('../data/processed/fig4_modelling/tf_tf_pairs/expr_GDSD6_pro_pro_vocab_info.csv')
# print(vocab_df.shape)
# vocab_df[:5]
TISSUE = 'SCC13-CTRLi'#'CAL27-CTRLi'#'A431-CTRLi'
type_enrich = 'expr_'
%%time
# filtered so that at least 10 genomic instances of vocab pair in significantly found in expressed genes 
results_expr_pro_pro = get_hits('../data/processed/fig4_modelling/tf_tf_pairs_scc/'+type_enrich+TISSUE+'_pro_pro_vocab_info.csv')

%%time
### TAKES AWHILE
results_expr_loop_loop = get_hits('../data/processed/fig4_modelling/tf_tf_pairs_scc/'+type_enrich+TISSUE+'_loop_loop_vocab_info.csv')<jupyter_output>**** reading ../data/processed/fig4_modelling/tf_tf_pairs_scc/expr_SCC13-CTRLi_loop_loop_vocab_info.csv
(51440, 15)
num_hits 4762
CPU times: user 19.9 s, sys: 53.6 ms, total: 20 s
Wall time: 20 s<jupyter_text>3B raw statistics (pre genomic instance filter)<jupyter_code>motifs_pro_pro = sorted(set(list(results_expr_pro_pro.vocab1.unique())+list(results_expr_pro_pro.vocab2.unique())))
print('num motifs in pro_pro', len(motifs_pro_pro))
print(motifs_pro_pro)


motifs_loop_loop = sorted(set(list(results_expr_loop_loop.vocab1.unique())+list(results_expr_loop_loop.vocab2.unique())))
print('num motifs in loop_loop', len(motifs_loop_loop))
print(motifs_loop_loop)
print('pre genomic instance filter')
print('num vocab in expression enrichment (pro-pro region): ', results_expr_pro_pro.vocab_pair.unique().shape[0])
print('num vocab in expression enrichment (loop-loop region): ', results_expr_loop_loop.vocab_pair.unique().shape[0])<jupyter_output>pre genomic instance filter
num vocab in expression enrichment (pro-pro region):  4757
num vocab in expression enrichment (loop-loop region):  2396<jupyter_text>3C. filter expression enriched vocab words if then have at least 10 genomic instances then get stats<jupyter_code>results_expr_pro_pro = filter_results(results_expr_pro_pro,min_hits_per_vocab=10)
# print(results_expr_pro_pro.vocab_pair.unique().shape)
# results_expr_pro_pro.vocab_pair.unique()
results_expr_loop_loop = filter_results(results_expr_loop_loop,min_hits_per_vocab=10)
# print(results_expr_loop_loop.vocab_pair.unique().shape)
# results_expr_loop_loop.vocab_pair.unique()
print('post genomic instance filter')
print('num vocab in expression enrichment (pro-pro region): ', results_expr_pro_pro.vocab_pair.unique().shape[0])
print(results_expr_pro_pro.vocab_pair.unique())
print('num vocab in expression enrichment (loop-loop region): ', results_expr_loop_loop.vocab_pair.unique().shape[0])
print(results_expr_loop_loop.vocab_pair.unique())<jupyter_output>post genomic instance filter
num vocab in expression enrichment (pro-pro region):  970
['ARNT::KLF3' 'ARNT::PATZ1' 'ARNT::SP1' 'CBFB::NFAT5' 'CEBPZ::KLF3'
 'CEBPZ::PATZ1' 'CEBPZ::SP1' 'E2F1::KLF3' 'E2F1::KLF5' 'E2F1::KLF6'
 'E2F1::PATZ1' 'E2F1::SP1' 'E2F1::SP3' 'E2F1::ZFX' 'E2F1::ZNF148'
 'E2F1::ZNF219' 'E2F1::ZNF281' 'E2F4::WT1' 'E2F4::ZFX' 'E2F4::ZNF219'
 'E2F6::EGR1' 'E2F6::FLI1' 'E2F6::KLF16' 'E2F6::KLF3' 'E2F6::KLF4'
 'E2F6::KLF5' 'E2F6::KLF6' 'E2F6::MAZ' 'E2F6::NFAT5' 'E2F6::PATZ1'
 'E2F6::SP1' 'E2F6::SP3' 'E2F6::SP4' 'E2F6::TAF1' 'E2F6::VEZF1'
 'E2F6::WT1' 'E2F6::ZBTB17' 'E2F6::ZFX' 'E2F6::ZNF148' 'E2F6::ZNF219'
 'E2F6::ZNF263' 'E2F6::ZNF281' 'E2F6::ZNF770' 'E2F7::WT1' 'E2F7::ZFX'
 'E2F7::ZNF148' 'E2F7::ZNF219' 'EGR1::ELF1' 'EGR1::ELK1' 'EGR1::FLI1'
 'EGR1::KLF16' 'EGR1::KLF3' 'EGR1::KLF4' 'EGR1::KLF5' 'EGR1::KLF6'
 'EGR1::MAZ' 'EGR1::NFAT5' 'EGR1::NR2F1' 'EGR1::PATZ1' 'EGR1::PURA'
 'EGR1::SP1' 'EGR1::SP2' 'EGR1::SP3' 'EGR1::SP4' 'EGR1::TAF1' 'EGR1::TBX1'
 'EGR1::TFAP2A' 'EGR1::[...]<jupyter_text>3D filter expr vocab words based on whether there is they are annotated for skin<jupyter_code>results_expr_pro_pro[['vocab_pair','check_tissuetf']].drop_duplicates().check_tissuetf.value_counts()
results_expr_loop_loop[['vocab_pair','check_tissuetf']].drop_duplicates().check_tissuetf.value_counts()<jupyter_output><empty_output><jupyter_text>shows fraction of vocab words having tissue specific motif configuration (`both` are tissue specific, `one` is tissue specific, or `none` are)MAKE SURE s are around 40 ish so pick `one` or `both`<jupyter_code>filter_thres = ['both']#'both' # 'one'#'none','one',
results_expr_pro_pro_tissue = results_expr_pro_pro[results_expr_pro_pro.check_tissuetf.isin(filter_thres)]
print('pro-pro region')
print('total vocab:',results_expr_pro_pro.vocab_pair.unique().shape[0],'tissue annon vocab:', results_expr_pro_pro_tissue.vocab_pair.unique().shape[0])
print(results_expr_pro_pro_tissue.vocab_pair.unique())

results_expr_loop_loop_tissue = results_expr_loop_loop[results_expr_loop_loop.check_tissuetf.isin(filter_thres)]
print('loop-loop region')
print('total vocab:',results_expr_loop_loop.vocab_pair.unique().shape[0],'tissue annon vocab:', results_expr_loop_loop_tissue.vocab_pair.unique().shape[0])
print(results_expr_loop_loop_tissue.vocab_pair.unique())<jupyter_output>pro-pro region
total vocab: 970 tissue annon vocab: 244
['ARNT::SP1' 'CEBPZ::SP1' 'E2F1::KLF5' 'E2F1::KLF6' 'E2F1::SP1'
 'E2F1::SP3' 'E2F1::ZNF219' 'E2F4::WT1' 'E2F4::ZFX' 'E2F4::ZNF219'
 'E2F6::KLF5' 'E2F6::SP1' 'E2F7::WT1' 'E2F7::ZFX' 'E2F7::ZNF148'
 'E2F7::ZNF219' 'EGR1::KLF5' 'EGR1::SP1' 'EGR1::SP3' 'EGR3::KLF5'
 'EGR3::SP1' 'EGR3::SP3' 'EHF::SP3' 'ELF1::KLF5' 'ELF1::SP1' 'ELF1::SP3'
 'ELF2::KLF5' 'ELF2::SP1' 'ELF2::SP3' 'ELF3::KLF5' 'ELF3::SP1' 'ELF3::SP3'
 'ELK1::KLF4' 'ELK1::KLF5' 'ELK1::SP1' 'ELK1::SP3' 'ELK4::KLF4'
 'ELK4::KLF5' 'ELK4::SP1' 'ELK4::SP3' 'EPAS1::KLF5' 'ETS1::KLF5'
 'ETS1::SP1' 'ETS1::SP2' 'ETS1::SP3' 'ETS1::SP4' 'ETS1::WT1' 'ETV5::KLF5'
 'ETV6::KLF5' 'ETV6::SP1' 'FLI1::KLF5' 'FLI1::SP1' 'FLI1::SP3'
 'HIF1A::KLF5' 'HSF1::SP1' 'IRF3::SP1' 'IRF5::SP1' 'KLF16::KLF4'
 'KLF16::KLF5' 'KLF16::MAZ' 'KLF16::SNAI2' 'KLF16::SP1' 'KLF16::SP3'
 'KLF16::TAF1' 'KLF16::TFAP2C' 'KLF16::WT1' 'KLF16::ZBTB14'
 'KLF16::ZNF219' 'KLF3::KLF5' 'KLF4::KLF5' 'KLF4::KLF6' 'KLF4::MAZ'
 'KLF4[...]<jupyter_text>3E add in stability scores vocab pairs that pass the genomic instance filter and get genomic instance counts<jupyter_code>def get_counts(results_df, label):
    counts_df = pd.DataFrame(results_df.vocab_pair.value_counts())
    counts_df.columns = ['num_instance']
    counts_df['label']=label
    return counts_df
vocab_summary_df = pd.concat([# get_counts(results_stability_pro_pro, 'stability_pro'),
            get_counts(results_expr_pro_pro_tissue, 'expr_pro_tissue'),
#             get_counts(results_stability_loop_loop, 'stability_loop'),
            get_counts(results_expr_loop_loop_tissue, 'expr_loop_tissue')
        ],
    axis=0)
vocab_summary_df.index.set_names('vocab',inplace=True)
vocab_summary_df.reset_index(inplace=True)
vocab_summary_df = vocab_summary_df.groupby('vocab').agg({'num_instance':sum, 'label':'|'.join}).reset_index()
vocab_summary_df['tissue']=TISSUE
vocab_summary_df.label.value_counts()
# results_stability_loop_loop.vocab_pair.unique()
vocab_summary_df.to_csv(os.path.join(save_dir, TISSUE+'_vocab_summary.csv'))
print('saved:', os.path.join(save_dir, TISSUE+'_vocab_summary.csv'))<jupyter_output>saved: ../data/processed/fig4_modelling/vocab_sum/SCC13-CTRLi_vocab_summary.csv<jupyter_text>4. manual run all 3 tissue typeslook at the *vocab_summary.csvmanually pair down list to one xlsx`vocab_sum/scc_filtered_vocab_list.xlsx`<jupyter_code># !pip install UpSetPlot
import glob, os
import pandas as pd
from upsetplot import plot, from_memberships
import seaborn as sns; 
import matplotlib.pyplot as plt
plt.style.use('seaborn-paper')
from matplotlib import rc
rc('font',**{'family':'sans-serif','sans-serif':['Arial']})
# rc('font',**{'family':'serif','serif':['Times']})
# rc('text', usetex=True)
save_dir = '../data/processed/fig4_modelling/vocab_sum/'
# glob.glob( os.path.join(save_dir, '*_vocab_summary.csv'))
scc_df = pd.concat([pd.read_csv('../data/processed/fig4_modelling/vocab_sum/A431-CTRLi_vocab_summary.csv',index_col=0),
          pd.read_csv('../data/processed/fig4_modelling/vocab_sum/CAL27-CTRLi_vocab_summary.csv',index_col=0),
          pd.read_csv('../data/processed/fig4_modelling/vocab_sum/SCC13-CTRLi_vocab_summary.csv',index_col=0)],axis=0)
mm_df = pd.concat([pd.read_csv('../data/processed/fig4_modelling/vocab_sum/COLO-SCR-DMSO_vocab_summary.csv',index_col=0),
          pd.read_csv('../data/processed/fig4_modelling/vocab_sum/WM-SCR-DMSO_vocab_summary.csv',index_col=0)],axis=0)


all_cancer = pd.concat([scc_df, mm_df]).groupby('vocab').agg({'tissue': '|'.join,'num_instance':sum, 'label': lambda x: '|'.join(list(set(x)))})
tissue_counts = all_cancer.tissue.value_counts()
print(tissue_counts)
names = [x.split('|') for x in tissue_counts.index]
values = list(tissue_counts.values)

data_upset = from_memberships(names, data=values)
plot(data_upset)
plt.savefig(os.path.join(save_dir, 'vocabs_cancer_upset.pdf'))
scc_df_tissue3 = scc_df.groupby('vocab').agg({'tissue': '|'.join,'num_instance':sum, 'label': lambda x: '|'.join(list(set(x)))})
scc_df_tissue3 = scc_df_tissue3[scc_df_tissue3.tissue=='A431-CTRLi|CAL27-CTRLi|SCC13-CTRLi'].reset_index()
scc_df_tissue3['tissue']= 'cancer'
scc_df_tissue3
scc_df_tissue3.to_csv( os.path.join(save_dir, 'SCC_all_vocab_summary.csv'))
# normal_df = pd.read_csv('../data/processed/fig4_modelling/vocab_sum/GDSD6_vocab_summary.csv',index_col=0)
normal_df =pd.concat([pd.read_csv( '../data/processed/fig4_modelling/tf_tf_pairs/expr_GDSD0_pair_filt_unique.csv'),
    pd.read_csv( '../data/processed/fig4_modelling/tf_tf_pairs/expr_GDSD3_pair_filt_unique.csv')],axis=0)

normal_df = normal_df[normal_df.pval_bonf <0.05]
normal_df
len(sorted(set(normal_df.tf_key.values)))
len(set(scc_df_tissue3.vocab.values))
set(scc_df_tissue3.vocab.values).intersection(set(normal_df.tf_key.values))
from matplotlib_venn import venn2

 
# Second way
fig = plt.figure()
plt.rcParams['figure.facecolor'] = 'white'

venn2([set(scc_df_tissue3.vocab.values), set(normal_df.vocab.values)],set_labels = ('Group A', 'Group B'))
plt.savefig()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14936985304153966, 'fraction_numerical': 0.03043814550252514, 'mean_word_length': 6.157277759687398, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '355011', 'license': 'MIT', 'n_tokens_mistral': 9290, 'n_tokens_neox': 8914, 'n_words': 1156, 'path': 'notebooks/8D5_SCC_vocab_enrich.ipynb', 'repo_name': 'mguo123/pan_omics', 'size': '21979'}