<jupyter_start><jupyter_text>Gorkin et al. Biorxiv 2017<jupyter_code>df = pd.read_excel('Table_S2_mouse_target_predicitons.xlsx',header=1)
tads = df['tad']<jupyter_output><empty_output><jupyter_text>TADs  TADs to bed<jupyter_code>tads = set(tads)
tups = []
for i in tads:
    chrom = i.split(':')[0]
    start,stop = i.split(':')[-1].split('-')
    tups.append((chrom,start,stop))
tad_df = pd.DataFrame(tups)
tad_df.to_csv('/home/shared/Data/encode/mouse/TAD_positions_mm10.bed',sep='\t',header=False,index=False)
import matplotlib.pyplot as plt
import seaborn as sb
%matplotlib inline
tad_df[1] = tad_df[1].apply(int)
tad_df[2] = tad_df[2].apply(int)
sb.set_context('paper',font_scale=1.8)
plt.hist((tad_df[2]-tad_df[1])/1e6,bins=30,color='grey')
plt.xlabel('size (mbps)')
plt.ylabel('frequency')
plt.vlines(0.5,0,250,label="Marbach et al.")
plt.legend()
plt.vlines?
import numpy as np
np.mean((tad_df[2]-tad_df[1])/1e6)<jupyter_output><empty_output><jupyter_text>dictionary of TADs<jupyter_code>tad_dic = {}
tad_df_sorted = pd.read_csv('/home/shared/Data/encode/mouse/TAD_positions_mm10_sorted.bed',sep='\t',header=None)
for indx,r in enumerate(tad_df_sorted.iterrows()):
    pos = "{}:{}-{}".format(r[1][0],r[1][1],r[1][2])
    tad_dic[pos] = "TAD_{}".format(indx)<jupyter_output><empty_output><jupyter_text>mine enh within TADs<jupyter_code>enh_tads = pd.read_csv('/home/shared/Data/encode/mouse/enh_within_TAD.bed',sep='\t',header=None)
tids = []
for r in enh_tads.iterrows():
    pos = "{}:{}-{}".format(r[1][3],r[1][4],r[1][5])
    tids.append(tad_dic[pos])

enh_tads['tids'] = tids
del(enh_tads[3])
del(enh_tads[4])
del(enh_tads[5])
enh_tads_groups = enh_tads.groupby('tids')
tad_enh_counts = enh_tads_groups.count()
sb.set_context('paper',font_scale=1.8)
plt.hist(tad_enh_counts[0],bins=50,color='grey')
plt.xlabel('enhs per TADs')
plt.ylabel('frequency')<jupyter_output><empty_output><jupyter_text>mine genes within TADs<jupyter_code>gene_tads = pd.read_csv('/home/shared/Data/encode/mouse/pc_within_TAD.bed',sep='\t',header=None)
tids = []
for r in gene_tads.iterrows():
    pos = "{}:{}-{}".format(r[1][6],r[1][7],r[1][8])
    tids.append(tad_dic[pos])

gene_tads['tids'] = tids
del(gene_tads[6])
del(gene_tads[7])
del(gene_tads[8])
gene_tads_groups = gene_tads.groupby('tids')
tad_gene_counts = gene_tads_groups.count()
sb.set_context('paper',font_scale=1.8)
plt.hist(tad_gene_counts[0],bins=40,color='grey')
plt.xlabel('genes per TADs')
plt.ylabel('frequency')<jupyter_output><empty_output><jupyter_text>using all annotations<jupyter_code>gencode_mm10 = pd.read_csv('/home/shared/Data/gencode/mouse/mm10/gencode.vM10.annotation.gtf',sep='\t',header=None,skiprows=5)
for g in exp['gene_id']:
    if g in list(gencode_mm10[8]):
        print(g)
        break
eid_to_name = {}
for r in gencode_mm10.iterrows():
    eid = r[1][8].split('gene_id "')[-1].split('";')[0]
    name = r[1][8].split('; gene_name "')[-1].split('";')[0]
    gtype = r[1][8].split('; gene_type "')[-1].split('";')[0]
    pos = '{}:{}-{}'.format(r[1][0],r[1][3],r[1][4])
    strand = r[1][6]
    eid_to_name[eid] = [name,gtype,pos,strand]

annot = pd.DataFrame(eid_to_name)
annot = annot.transpose()
annot_subset = annot.ix[[e in list(exp['gene_id']) for e in annot.index],:]
annot_subset.groupby(by=1).count()
gencode_mm10 = pd.read_csv('/home/shared/Data/gencode/mouse/mm10/gencode.vM10.annotation.gtf',sep='\t',header=None,skiprows=5)
for g in exp['gene_id']:
    if g in list(gencode_mm10[8]):
        print(g)
        break
eid_to_name = {}
for r in gencode_mm10.iterrows():
    eid = r[1][8].split('gene_id "')[-1].split('";')[0]
    name = r[1][8].split('; gene_name "')[-1].split('";')[0]
    gtype = r[1][8].split('; gene_type "')[-1].split('";')[0]
    pos = '{}:{}-{}'.format(r[1][0],r[1][3],r[1][4])
    strand = r[1][6]
    eid_to_name[eid] = [name,gtype,pos,strand]

annot = pd.DataFrame(eid_to_name)
annot = annot.transpose()
annot_subset = annot.ix[[e in list(exp['gene_id']) for e in annot.index],:]<jupyter_output><empty_output><jupyter_text>only protein coding<jupyter_code>gencode_m4 = pd.read_csv('/home/shared/Data/gencode/mouse/M4/protein_coding.gtf',sep='\t',header=None)
eid_to_name = {}
for indx,r in enumerate(gencode_m4.iterrows()):
    eid = r[1][8].split('gene_id "')[-1].split('";')[0]
    name = r[1][8].split('; gene_name "')[-1].split('";')[0]
    gtype = r[1][8].split('; gene_type "')[-1].split('";')[0]
    pos = '{}:{}-{}'.format(r[1][0],r[1][3],r[1][4])
    strand = r[1][6]
    eid_to_name[eid] = [name,gtype,pos,strand,indx]

annot = pd.DataFrame(eid_to_name)
annot = annot.transpose()
annot_subset = annot.ix[[e in list(exp['gene_id']) for e in annot.index],:]
annot_subset.groupby(by=1).count()<jupyter_output><empty_output><jupyter_text>expression data<jupyter_code>exp = pd.read_csv('/home/shared/Data/encode/mouse/gene_expression_log2TPM_signal_matrix.txt',sep='\t')
tissue_of_interest = ['lung','kidney','stomach','intestine','embryonic-facial-prominence','neural-tube','limb','midbrain','forebrain','liver','hindbrain','heart']
time = ['P0','e11.5','e12.5','e13.5','e14.5','e15.5','e16.5']
exp.set_index('gene_id',inplace=True)
exp = exp.ix[:,[any(st in c for st in tissue_of_interest) and any(t in c for t in time) for c in exp.columns]]
exp.reset_index(inplace=True)
count = 0
pos_tup = []
eid_name_tup = []
tup = []
for g in exp['gene_id']:
    if g in eid_to_name.keys():
        tup.append((eid_to_name[g][0],g))
eid_gene = pd.DataFrame(tup)
exp_pc = exp.ix[[g in list(eid_gene[1]) for g in exp['gene_id']],:]
exp_pc.reset_index(drop=True,inplace=True)
exp_pc['name'] = list(eid_gene[0])
del(exp_pc['gene_id'])
exp_pc_df = exp_pc.set_index('name')
exp_pc_df = exp_pc_df[~exp_pc_df.index.duplicated()]
exp_pc_df_corr = exp_pc_df.corr()
exp_pc_df_corr.index
tissue_col = []
for s in exp_pc_df_corr.columns:
    tissue_col.append(s.split('_')[0])
time_row = []
for s in exp_pc_df_corr.index:
    time_row.append(s.split('_')[1])
    
import matplotlib

def get_color(arr, color = 'jet'):
    cmap = matplotlib.cm.get_cmap(color,len(set(arr)))
    out_arr = []
    for val in arr:
        if val is None:
            out_arr.append('black')
        else:
            out_arr.append(cmap(val))
    return(out_arr)
tissue_zip = zip(set(tissue_col),range(0,len(set(tissue_col))))
time_zip = zip(set(time_row),range(0,len(set(time_row))))
tissue_num = {}
for v in list(tissue_zip):
    tissue_num[v[0]] = v[1]
time_num = {}
for v in list(time_zip):
    time_num[v[0]] = v[1]
tissue_col_value = []
for t in tissue_col:
    tissue_col_value.append(tissue_num[t])
    
time_row_value = []
for t in time_row:
    time_row_value.append(time_num[t])
times = ["P0","e11.5","e12.5","e13.5","e14.5","e15.5","e16.5"]
tissues = ["embryonic-facial-prominence","midbrain","forebrain","hindbrain","neural-tube","kidney","lung","liver","intestine","stomach","heart","limb"]
col_pal = sb.color_palette("hls",len(set(tissue_col)))
row_pal = sb.color_palette("BuGn_r",len(set(time_row)))
col_lut = dict(zip(map(str,tissues),col_pal))
row_lut = dict(zip(map(str,times),row_pal))
col_color_bar = [col_lut[l] for l in tissue_col]
row_color_bar = [row_lut[l] for l in time_row]
exp_clust = sb.clustermap(exp_pc_df.corr(),col_colors=col_color_bar,row_colors=row_color_bar,vmin=0,vmax=1,xticklabels=False,yticklabels=False,cmap='Blues')
for label in tissues:
    exp_clust.ax_col_dendrogram.bar(0,0,color=col_lut[label],label=label,linewidth=0)
for label in times:
    exp_clust.ax_row_dendrogram.bar(0,0,color=row_lut[label],label=label,linewidth=0)
exp_clust.ax_col_dendrogram.legend(loc=(0,1),ncol=3)
exp_clust.ax_row_dendrogram.legend(loc=(1.26,1.55),ncol=3)
    
exp_clust.data2d
filter_r = []
for e in exp_pc_df['gene_id']:
    filter_r.append(eid_to_name[e][-1])
m4_dups_filter = gencode_m4.ix[filter_r,:]
m4_dups_filter.reset_index(drop=True,inplace=True)
m4_dups_filter.to_csv('/home/shared/Data/gencode/mouse/M4/protein_coding_filtered.gtf',sep='\t',index=None,header=None,columns=None)
m4_dups_filter_bed = m4_dups_filter[[0,3,4,8,5,6]]
gname = []
for r in m4_dups_filter_bed.iterrows():
    name = r[1][8].split('; gene_name "')[-1].split('";')[0]
    gname.append(name)
m4_dups_filter_bed[8] = gname
m4_dups_filter_bed.to_csv('/home/shared/Data/encode/mouse/m4_protein_coding_genes.bed',sep='\t',index=None,header=None)<jupyter_output><empty_output><jupyter_text>Scratch<jupyter_code>import pandas as pd
A = pd.DataFrame(0,index=['A','B','C','D','E','F'],columns=['i','j','k','l','m'])
import numpy as np

a_s = pd.DataFrame(np.random.sample(size=(2,3)))
a_s.index = ['C','D']
a_s.columns = ['i','k','m']
a_s_dic = a_s.to_dict()
for c in a_s_dic.keys():
    for r in a_s_dic[c].keys():
        A_dic[c][r] = a_s_dic[c][r]
pd.DataFrame(A_dic)
A_dic = A.to_dict()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20787920072661217, 'fraction_numerical': 0.028723887375113534, 'mean_word_length': 6.490646258503402, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '215430', 'license': 'MIT', 'n_tokens_mistral': 4049, 'n_tokens_neox': 3888, 'n_words': 472, 'path': 'enh_gene/ENCODE_mouse_enh-gene_overlapping_tads.ipynb', 'repo_name': 'chbe-helix/TreNCo', 'size': '8808'}