<jupyter_start><jupyter_text>This file is part of MADIP: Molecular Atlas Data Integration PipelineThis file provide some additional nomenclature refinements and concentrations calculationCopyright 2021 Blue Brain Project / EPFL    Licensed under the Apache License, Version 2.0 (the "License");   you may not use this file except in compliance with the License.   You may obtain a copy of the License at       http://www.apache.org/licenses/LICENSE-2.0   Unless required by applicable law or agreed to in writing, software   distributed under the License is distributed on an "AS IS" BASIS,   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   See the License for the specific language governing permissions and   limitations under the License.<jupyter_code>import pandas as pd
import numpy as np

from matplotlib import pyplot as plt
import seaborn as sns

from itertools import chain

import re

import pickle as pkl

pd.options.display.max_columns = None
pd.options.display.max_rows = None
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
sns.set_style("whitegrid")
with open('../data/2_df_best_alignedIDs_9May2021.pkl','rb') as f:
    df = pkl.load(f)
print(len(df))<jupyter_output>2132553<jupyter_text>Some additional checks and refinements<jupyter_code>df.loc[df['gene_id_final'].str.contains(';')]
df.loc[df['gene_id_final']=='C2CD4CC2CD4FAMILY','gene_id_final'] = 'C2CD4CC2CD4'
df.loc[df['gene_id_final'].isna()]
df.loc[df['raw_data']==0]
df.loc[df['raw_data']<0]
#for figures
df['log_raw_data'] = np.log(df['raw_data'])
print(len(df['raw_data_units'].unique()))
df['raw_data_units'].unique()
df.loc[df['raw_data_units']=='Protein Abundance (Summerized TMT Reporter Ion Intensities)','raw_data_units'] = 'tmt abundance'

df.loc[df['raw_data_units']=='LFQ','raw_data_units'] = 'LFQintensity'

print(len(df['raw_data_units'].unique()))
df['raw_data_units'].unique()
len(df.loc[(df['Uniprot_unified'].isna()) & (df['uniprot_from_gn'] == 'NoMapping')])
df.loc[(df['Uniprot_unified'].isna()) & (df['uniprot_from_gn'] == 'NoMapping'),'Study'].unique()
df.loc[df['Study']=='Duda 2018','raw_data_units'].unique()
df.loc[df['Study']=='Kjell 2020','raw_data_units'].unique()
df.loc[df['Study']=='Carlyle 2017','raw_data_units'].unique()
df.loc[df['Study']=='Fecher 2019','raw_data_units'].unique()
df.loc[df['Study']=='Han 2014','raw_data_units'].unique()<jupyter_output><empty_output><jupyter_text>Check Uniprot to get protein seq to count number of possible peptides<jupyter_code># Uniprot 21july2020

#(taxonomy:"Mus musculus (Mouse) [10090]" OR taxonomy:"Rattus norvegicus (Rat) [10116]" OR taxonomy:"Homo sapiens (Human) [9606]") AND reviewed:yes
#(taxonomy:"Mus musculus (Mouse) [10090]" OR taxonomy:"Rattus norvegicus (Rat) [10116]" OR taxonomy:"Homo sapiens (Human) [9606]") AND reviewed:no


uniprot_rev = pd.read_csv('../data/uniprot_rev_taxonomyMRH_21july2020.tab', sep='\t')
print(len(uniprot_rev))

uniprot_unrev = pd.read_csv('../data/uniprot_taxonomyMRH_unreviewed_21july2020.gz', sep='\t')
print(len(uniprot_unrev))

uniprot_rev.loc[uniprot_rev['Entry'].str.contains("-")].head()
uniprot_unrev.loc[uniprot_unrev['Entry'].str.contains("-")].head()
uniprot_rev = uniprot_rev.loc[~uniprot_rev['Gene names'].isna()]
uniprot_unrev = uniprot_unrev.loc[~uniprot_unrev['Gene names'].isna()]
print(len(uniprot_rev))
print(len(uniprot_unrev))

uniprot_rev['Gene names'] = uniprot_rev['Gene names'].str.upper()
uniprot_unrev['Gene names'] = uniprot_unrev['Gene names'].str.upper()
s1 = uniprot_rev['Gene names'].str.split(' ').apply(pd.Series, 1).stack()
s1.index = s1.index.droplevel(-1) 
s1.name = 'Gene names' 
del uniprot_rev['Gene names']
uniprot_rev = uniprot_rev.join(s1)


s2 = uniprot_unrev['Gene names'].str.split(' ').apply(pd.Series, 1).stack()
s2.index = s2.index.droplevel(-1) 
s2.name = 'Gene names' 
del uniprot_unrev['Gene names']
uniprot_unrev = uniprot_unrev.join(s2)
len(uniprot_rev['Gene names'].unique())/len(uniprot_rev)
uniprot_rev.columns
uniprot_rev = uniprot_rev.drop(columns = ['Entry name', 'Status', 'Protein names', 'Organism', 'Length'])
uniprot_unrev = uniprot_unrev.drop(columns = ['Entry name', 'Status', 'Protein names', 'Organism', 'Length'])
uniprot_rev = pd.DataFrame(uniprot_rev.groupby('Gene names')['Entry'].apply(list))
uniprot_unrev = pd.DataFrame(uniprot_unrev.groupby('Gene names')['Entry'].apply(list))
uniprot_rev_dict = pd.Series(uniprot_rev['Entry'].values,index=uniprot_rev.index).to_dict() 
uniprot_unrev_dict = pd.Series(uniprot_unrev['Entry'].values,index=uniprot_unrev.index).to_dict() 


uniprot_rev_genes = list(set([item for sublist in uniprot_rev.index.tolist() for item in sublist]))
uniprot_unrev_genes = list(set([item for sublist in uniprot_unrev.index.tolist() for item in sublist]))
uniprot_rev_ids = list(set([item for sublist in uniprot_rev['Entry'].values.tolist() for item in sublist]))
uniprot_unrev_ids = list(set([item for sublist in uniprot_unrev['Entry'].values.tolist() for item in sublist]))
# Data downloaded on 05june2020 is from 
# ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/

mouse_uniprot_ids = pd.read_csv('../data/MOUSE_10090_idmapping.dat.gz',header=None,sep='\t')
rat_uniprot_ids = pd.read_csv('../data/RAT_10116_idmapping.dat.gz',header=None,sep='\t')
human_uniprot_ids = pd.read_csv('../data/HUMAN_9606_idmapping.dat.gz',header=None,sep='\t')

mouse_uniprot_ids.columns = ['Uniprot','ID_type','ID']
rat_uniprot_ids.columns = ['Uniprot','ID_type','ID']
human_uniprot_ids.columns = ['Uniprot','ID_type','ID']

mouse_uniprot_ids['id_of_organism'] = 'mouse'
rat_uniprot_ids['id_of_organism'] = 'rat'
human_uniprot_ids['id_of_organism'] = 'human'

#combine data for multiple organisms
uniprot_ids_mrh = pd.concat([mouse_uniprot_ids,rat_uniprot_ids,human_uniprot_ids],ignore_index=True,sort=True)

print((len(mouse_uniprot_ids['Uniprot'].unique())+len(rat_uniprot_ids['Uniprot'].unique())+len(human_uniprot_ids['Uniprot'].unique()))/len(uniprot_ids_mrh['Uniprot'].unique()),len(uniprot_ids_mrh))

#keep only needed id-types
print(len(uniprot_ids_mrh))
uniprot_ids_mrh = uniprot_ids_mrh.loc[uniprot_ids_mrh['ID_type'].isin(['UniProtKB-ID', 'Gene_Name','GeneID','Gene_Synonym','GeneCards','HGNC'])].copy()
print(len(uniprot_ids_mrh))

uniprot_ids_mrh = uniprot_ids_mrh.reset_index(drop=True)

uniprot_ids_mrh['ID'] = uniprot_ids_mrh['ID'].str.upper()

len(uniprot_ids_mrh.loc[uniprot_ids_mrh['ID_type']=='Gene_Name','Uniprot'].unique())
uniprot_ids_mrh_gnDupl = uniprot_ids_mrh.loc[uniprot_ids_mrh['ID_type']=='Gene_Name',['Uniprot','ID']]

print(len(uniprot_ids_mrh_gnDupl))
uniprot_ids_mrh_gnDupl = uniprot_ids_mrh_gnDupl.drop_duplicates(keep=False)

print(len(uniprot_ids_mrh_gnDupl))
uniprot_ids_mrh_gnDupl.head(10)
uniprot_gn = uniprot_ids_mrh.loc[uniprot_ids_mrh['ID_type']=='Gene_Name',['Uniprot','ID']].groupby('Uniprot').aggregate(lambda tdf: tdf.unique().tolist())

uniprot_ids_mrh_dict = pd.Series(uniprot_gn['ID'].values,index=uniprot_gn.index).to_dict()
gn_uniprot = uniprot_ids_mrh.loc[uniprot_ids_mrh['ID_type']=='Gene_Name',['Uniprot','ID']].groupby('ID').aggregate(lambda tdf: tdf.unique().tolist())

uniprot_gn_mrh_dict =  pd.Series(gn_uniprot['Uniprot'].values,index=gn_uniprot.index).to_dict() 
print(len(df))
print(len(df.loc[~((df['Uniprot_unified'].isna()) & (df['uniprot_from_gn'] == 'NoMapping'))]))  
print(len(df.loc[((df['Uniprot_unified'].isna()) & (df['uniprot_from_gn'] == 'NoMapping'))]))
print(len(df.loc[~df['Uniprot_unified'].isna()]))
print(len(df.loc[df['Uniprot_unified'].isna()]))
print(len(df.loc[df['Uniprot'].isna()]))
len(df.loc[(~df['Uniprot_unified'].isna()) & (df['Uniprot_unified'].str.contains('CON_'))])
df.loc[(~df['Uniprot_unified'].isna()) & (df['Uniprot_unified'].str.contains('CON_'))].head()<jupyter_output><empty_output><jupyter_text>Given<jupyter_code># gn to uniprot

uniprot_rev_dict["VIM"]
# gn to uniprot

uniprot_unrev_dict["VIM"]
# gn to list of uniprots, based on ftp mapping file

uniprot_gn_mrh_dict['VIM']
df.columns<jupyter_output><empty_output><jupyter_text>Needed<jupyter_code>df.columns
df_fgn = df[['gene_id_final', 'Uniprot', 'Uniprot_unified']].copy()

print(len(df_fgn))

df_fgn = df_fgn.drop_duplicates(keep='first')
df_fgn = df_fgn.reset_index(drop=True)

print(len(df_fgn))

df_fgn['uniprot_from_gn'] = df_fgn['gene_id_final'].copy()

df_fgn['uniprot_from_gn'] = df_fgn['uniprot_from_gn'].map(uniprot_rev_dict).fillna(df_fgn['uniprot_from_gn'].map(uniprot_unrev_dict).fillna(df_fgn['uniprot_from_gn'].map(uniprot_gn_mrh_dict).fillna('NoMapping')))  #

df_fgn.head()
print(len(df_fgn.loc[df_fgn['uniprot_from_gn']=='NoMapping']))
len(df_fgn.loc[df_fgn['uniprot_from_gn']=='NoMapping'])/len(df_fgn)
df_fgn.loc[(df_fgn['uniprot_from_gn']=='NoMapping') & (df_fgn['Uniprot'].isna())]
len(df_fgn.loc[( (df_fgn['uniprot_from_gn']=='NoMapping') & (df_fgn['Uniprot'].isna()) ) ])
df_fgn.loc[( (df_fgn['uniprot_from_gn']=='NoMapping') & (df_fgn['Uniprot'].isna()) ) ]
df_fgn.loc[( (df_fgn['uniprot_from_gn']=='NoMapping') & (df_fgn['Uniprot_unified'].isna()) ) ]
print(len(df_fgn))
df_fgn = df_fgn.loc[~( (df_fgn['uniprot_from_gn']=='NoMapping') & (df_fgn['Uniprot'].isna()) ) ]
print(len(df_fgn))
df_fgn.loc[df_fgn['uniprot_from_gn']=='NoMapping'].head()
len(df_fgn.loc[(~df_fgn['Uniprot'].isna()) & (df_fgn['Uniprot'].str.contains("-"))])
len(df_fgn.loc[(df_fgn['Uniprot'].isna())])
len(df_fgn.loc[(df_fgn['Uniprot'].isna()) & (df_fgn['uniprot_from_gn'] != "NoMapping")] )
len(df_fgn.loc[(df_fgn['Uniprot'].isna()) & (~df_fgn['Uniprot_unified'].isna())])
len(df_fgn.loc[(~df_fgn['Uniprot'].isna()) & (df_fgn['Uniprot_unified'].isna())])
len(df_fgn.loc[(df_fgn['Uniprot_unified'].isna())])<jupyter_output><empty_output><jupyter_text>Uniprot_final1. Uniprot isna & uniprot_from_gn != "NoMapping" -> Uniprot_final = uniprot_from_gn[0]1.5. Uniprot isna & uniprot_from_gn == "NoMapping" -> Uniprot_final = np.nan2. Uniprot_unified in Uniprot -> Uniprot_final = Uniprot_unified3. uniprot_from_gn == NoMapping -> Uniprot_final = Uniprot.split(";")[0].split("-")[0] 4. any(uniprot_from_gn) in Uniprot -> Uniprot_final = uniprot_from_gn which is in Uniprot 5. Uniprot_unified in uniprot_from_gn -> Uniprot_final = Uniprot_unified (attention, check for consistency)<jupyter_code>#check if at least one item in list exists in another list
any_in = lambda a, b: any(i in b for i in a)

def get_uniprot_final(index,row):
    
    uniprot = row['Uniprot']
    uniprot_from_gn = row['uniprot_from_gn']
    
    
    #case 1
    if (isinstance(uniprot,float))&(uniprot_from_gn!="NoMapping"):
        return uniprot_from_gn[0].split("-")[0]
    
    #case 1.5
    elif (isinstance(uniprot,float))&(uniprot_from_gn=="NoMapping"):
        return np.nan
    
    else:
        uniprots_list0 = uniprot.replace(" ","").split(";")
        uniprots_list = [x.split("-")[0] for x in uniprots_list0 if x is not None] 
        
        uniprot_unified = row['Uniprot_unified'].split("-")[0]
    
        
        #case 2
        if uniprot_unified in uniprots_list:
            return uniprot_unified
        
        #case 3
        elif uniprot_from_gn=="NoMapping":
            return uniprots_list[0]
            
        #case 4
        elif any_in(uniprot_from_gn,uniprots_list):
            ugn = [x for x in uniprot_from_gn if x in uniprots_list]
            if isinstance(ugn,list):
                if len(ugn)>0:
                    return ugn[0]
            else:
                print("attention #################",index)
            
        #case 5
        elif uniprot_unified in uniprot_from_gn:
            #print("check ",index)
            return uniprot_unified
        
        #case 6
        else:
            if any_in(uniprots_list,uniprot_rev_ids):
                ugn1 = [x for x in uniprots_list if x in uniprot_rev_ids]
                if isinstance(ugn1,list):
                    if len(ugn1)>0:
                        return ugn1[0]
                else:
                    print("attention #################",index)

                
            elif any_in(uniprots_list,uniprot_unrev_ids): 
                ugn2 = [x for x in uniprots_list if x in uniprot_unrev_ids]
                if isinstance(ugn2,list):
                    if len(ugn2)>0:
                        return ugn2[0]
                else:
                    print("attention #################",index)
                    
                
            else:
                return "#".join(['attention',uniprot_from_gn[0]])
                #print("ATTENTTION",index)
                # make counts dict and return the most common
                #return "#".join(['attention',uniprots_list[0]])
# need Uniprot to get protein seq to count number of possible peptides

#df_fgn = df_fgn.drop(columns='Uniprot_final')

df_fgn = df_fgn.reset_index(drop=True)

df_fgn['Uniprot_final'] =  None

for index,row in df_fgn.iterrows():
    #print(index)
    df_fgn.loc[index,'Uniprot_final'] = get_uniprot_final(index,row)
len(df_fgn.loc[df_fgn['Uniprot_final'].str.contains("attention")])<jupyter_output><empty_output><jupyter_text>Manually check remained ids<jupyter_code>df_fgn.loc[(df_fgn['gene_id_final']=='SLC8A1') & (df_fgn['Uniprot_final']=='attention#P32418'),'Uniprot_final'] = "P32418" 

df_fgn.loc[(df_fgn['gene_id_final']=='GM15800') & (df_fgn['Uniprot_final']=='attention#Q6GQX8'),'Uniprot_final'] = "Q6GQX8" 

df_fgn.loc[(df_fgn['gene_id_final']=='TPR') & (df_fgn['Uniprot_final']=='attention#P12270'),'Uniprot_final'] = "P12270" 

df_fgn.loc[(df_fgn['gene_id_final']=='TSC2') & (df_fgn['Uniprot_final']=='attention#P49816'),'Uniprot_final'] = "P49816" 

df_fgn.loc[(df_fgn['gene_id_final']=='ARMCX4') & (df_fgn['Uniprot_final']=='attention#Q5H9R4'),'Uniprot_final'] = "Q5H9R4" 

df_fgn.loc[(df_fgn['gene_id_final']=='IFI204') & (df_fgn['Uniprot_final']=='attention#P0DOV2'),'Uniprot_final'] = "P0DOV2"

df_fgn.loc[(df_fgn['gene_id_final']=='DOS') & (df_fgn['Uniprot_final']=='attention#Q66L44'),'Uniprot_final'] = "Q66L44" # DOS is synonymous GN in Q66L44 entry

df_fgn.loc[(df_fgn['gene_id_final']=='MEX3A') & (df_fgn['Uniprot_final']=='attention#A1L020'),'Uniprot_final'] = "A1L020" 

df_fgn.loc[(df_fgn['gene_id_final']=='EVI5L') & (df_fgn['Uniprot_final']=='attention#Q96CN4'),'Uniprot_final'] = "Q96CN4" 

df_fgn.loc[(df_fgn['gene_id_final']=='RCOR1') & (df_fgn['Uniprot_final']=='attention#Q9UKL0'),'Uniprot_final'] = "Q9UKL0"

df_fgn.loc[(df_fgn['gene_id_final']=='NAGLU') & (df_fgn['Uniprot_final']=='attention#P54802'),'Uniprot_final'] = "P54802" 

df_fgn.loc[(df_fgn['gene_id_final']=='TRPM3') & (df_fgn['Uniprot_final']=='attention#Q9HCF6'),'Uniprot_final'] = "Q9HCF6" 

df_fgn.loc[(df_fgn['gene_id_final']=='FXR2') & (df_fgn['Uniprot_final']=='attention#Q9WVR4'),'Uniprot_final'] = "Q9WVR4" 

df_fgn.loc[(df_fgn['gene_id_final']=='OSBPL10') & (df_fgn['Uniprot_final']=='attention#S4R1M9'),'Uniprot_final'] = "S4R1M9" 

df_fgn.loc[(df_fgn['gene_id_final']=='SAP18') & (df_fgn['Uniprot_final']=='attention#O55128'),'Uniprot_final'] = "O55128" 

df_fgn.loc[(df_fgn['gene_id_final']=='HLA-A') & (df_fgn['Uniprot_final']=='attention#P04439'),'Uniprot_final'] = "P04439"

df_fgn.loc[(df_fgn['gene_id_final']=='HLA-C') & (df_fgn['Uniprot_final']=='attention#P10321'),'Uniprot_final'] = "P10321" 

df_fgn.loc[(df_fgn['gene_id_final']=='C21ORF33') & (df_fgn['Uniprot_final']=='attention#P0DPI2'),'Uniprot_final'] = "P0DPI2" # C21ORF33 is synonymous GN in Uniprot

df_fgn.loc[(df_fgn['gene_id_final']=='HMG1L1') & (df_fgn['Uniprot_final']=='attention#B2RPK0'),'Uniprot_final'] = "B2RPK0" 

df_fgn.loc[(df_fgn['gene_id_final']=='SRP54C') & (df_fgn['Uniprot_final']=='attention#Q99JZ9'),'Uniprot_final'] = "Q99JZ9" 

df_fgn.loc[(df_fgn['gene_id_final']=='SYNJ2BP-COX16') & (df_fgn['Uniprot_final']=='attention#A0A087WYV9'),'Uniprot_final'] = "A0A087WYV9" 

df_fgn.loc[(df_fgn['gene_id_final']=='PPP1R2P3') & (df_fgn['Uniprot_final']=='attention#Q6NXS1'),'Uniprot_final'] = "Q6NXS1" 

df_fgn.loc[(df_fgn['gene_id_final']=='5330417C22RIK') & (df_fgn['Uniprot_final']=='attention#A0A0A0MQC6'),'Uniprot_final'] = "A0A0A0MQC6"



df_fgn.loc[df_fgn['Uniprot_final'].str.contains("attention")]

df_fgn.loc[df_fgn['Uniprot_final'].isna()]
# in some applications ca be useful:

#df['Uniprot_final'] = None

#df.loc[df['Uniprot_unified'].isna(), 'Uniprot_final'] = df.loc[df['Uniprot_unified'].isna(),'uniprot_from_gn']
#df.loc[~df['Uniprot_unified'].isna(), 'Uniprot_final'] = df.loc[~df['Uniprot_unified'].isna(),'Uniprot_unified']


#df.loc[df['Uniprot_unified'].isna(),'Uniprot_final'] = df.loc[df['Uniprot_unified'].isna(),'uniprot_from_gn'].map(lambda x: x[0])
# clean data

print(df.loc[df['Uniprot_unified'].isin(['D3YYU8D3Z0M8','P01900P14427','P47963Q5RKP3','P63242Q8BGY2','Q3TCJ1D3Z4D8','Q6ZWZ6P63323','Q8CAY6Q80X81','Q8CHF5Q9JKP5','Q91WK5Q9CV53','Q9CQK7E9PXV5','Q9CR27Q9CYF6','Q9DCS2E9Q5B2','Q9JM76D3Z2F8']),'Study'].unique())

df_fgn.loc[df_fgn['Uniprot_final']=='D3YYU8D3Z0M8','Uniprot_final'] = 'D3YYU8'
df_fgn.loc[df_fgn['Uniprot_final']=='P01900P14427','Uniprot_final'] = 'P01900'
df_fgn.loc[df_fgn['Uniprot_final']=='P47963Q5RKP3','Uniprot_final'] = 'P47963'
df_fgn.loc[df_fgn['Uniprot_final']=='P63242Q8BGY2','Uniprot_final'] = 'P63242'
df_fgn.loc[df_fgn['Uniprot_final']=='Q3TCJ1D3Z4D8','Uniprot_final'] = 'Q3TCJ1'
df_fgn.loc[df_fgn['Uniprot_final']=='Q6ZWZ6P63323','Uniprot_final'] = 'Q6ZWZ6'
df_fgn.loc[df_fgn['Uniprot_final']=='Q8CAY6Q80X81','Uniprot_final'] = 'Q8CAY6'
df_fgn.loc[df_fgn['Uniprot_final']=='Q8CHF5Q9JKP5','Uniprot_final'] = 'Q8CHF5'
df_fgn.loc[df_fgn['Uniprot_final']=='Q91WK5Q9CV53','Uniprot_final'] = 'Q91WK5'
df_fgn.loc[df_fgn['Uniprot_final']=='Q9CQK7E9PXV5','Uniprot_final'] = 'Q9CQK7'
df_fgn.loc[df_fgn['Uniprot_final']=='Q9CR27Q9CYF6','Uniprot_final'] = 'Q9CR27'
df_fgn.loc[df_fgn['Uniprot_final']=='Q9DCS2E9Q5B2','Uniprot_final'] = 'Q9DCS2'
df_fgn.loc[df_fgn['Uniprot_final']=='Q9JM76D3Z2F8','Uniprot_final'] = 'Q9JM76'

df.loc[df['Uniprot_unified']=='P13864'].head(1)

df.loc[df['Uniprot_unified']=='Q76I79'].head(1)
df = df.drop(columns=['gn_from_uniprot', 'uniprot_from_gn'])
df = df.reset_index(drop=True)
print(len(df))
df_all = pd.merge(df,df_fgn,how='inner',on=['gene_id_final', 'Uniprot', 'Uniprot_unified'])
len(df_all)
extra = pd.merge(df,df_fgn, how='left', indicator=True)
extra.head()
extra['_merge'].unique()
len(extra.loc[extra['_merge']=='left_only'])
extra.loc[(extra['_merge']=='left_only') & (~extra['uniprot_from_gn'].isna()) & (~extra['Uniprot'].isna())]
df = df_all.copy()<jupyter_output><empty_output><jupyter_text>Theor pep number<jupyter_code>#Proteases

#trypsin:  (.*?(?:K|R|$))
#lysC:   (.*?(?:K|$))
df['TheorPepNum'] = None

# needed only for:
studiesTheorPepNum = ['Hamezah 2019', 'Sharma 2015, isolated', 'Guergues 2019','Sharma 2015, cultured', 
                      'Han 2014', 'Kjell 2020', 'Krogager 2018','Hamezah 2018', 
                      'Zhu 2018', 'Fecher 2019', 'Carlyle 2017','McKetney 2019',
                      'Geiger 2013','Hasan 2020', 'Bai 2020'] #,'Wisniewski 2015','Duda 2018'



# trypsin | trypsin + LysC
studiesTrypsin = ['Hamezah 2018','Krogager 2018','Sharma 2015, isolated','Sharma 2015, cultured','Han 2014','Hamezah 2019','Guergues 2019','Kjell 2020','Zhu 2018','Fecher 2019','Carlyle 2017','McKetney 2019','Hasan 2020','Bai 2020'] #,'Wisniewski 2015','Duda 2018']    

# 'Krogager 2018': Trypsin & L-Lys ### both (?) sequent.. for approx trypsin because of enzyme specificty
# 'Han 2014': use trypsin for approximation (from supp info: The proteins were first digested with trypsin (enzyme-to-substrate ratio [w/w] of 1:100) at 37°C overnight, after which the peptides were collected by centrifugation. In the second digestion, the filter units were washed sequentially with water, UA buffer, and 40 mM ammonium bicarbonate, respectively, and the proteins were cleaved with trypsin (enzyme-to-substrate ratio [w/w] of 1:200).)
# 'Geiger 2013':  digested in solution with endoprotease Lys-C. 

# 'Hamezah 2019': Trypsin was set as the digestive enzyme
# 'Guergues 2019': Trypsin/Lys-C protease - for approx trypsin because of enzyme specificty
# 'Kjell 2020': digested with LysC and trypsin - for approx trypsin because of enzyme specificty
# 'Zhu 2018': two-step digestion was performed at 37 °C with Lys-C and trypsin - for approx trypsin because of enzyme specificty
# 'Fecher 2019': LysC (Promega) followed by 4-h incubation with 0.15 μg trypsin - for approx trypsin because of enzyme specificty
# 'Carlyle 2017': Trypsin-digested peptides 
# 'McKetney 2019': Trypsin was added to the protein lysate sample at a ratio of 50:1 w/v and digested overnight.
# 'Hasan 2020': Sequencing-grade modified porcine trypsin 
# 'Bai 2020': Lys-C (Wako, 1:100 w/w) at 21 C for 2 h, diluted by 4-fold to reduce urea to 2 M for the addition of trypsin - for approx trypsin because of enzyme specificty

# 'Wisniewski 2015': Lys-C and trypsin were used for sequential digestion of proteins.
# using LysC and trypsin.

# lysC
studiesLysC = ['Geiger 2013']

#

uniprot_TP = df.loc[df['Study'].isin(studiesTheorPepNum),'Uniprot_final'].unique().tolist()

uniprot_tryp = df.loc[df['Study'].isin(studiesTrypsin),'Uniprot_final'].unique().tolist()
uniprot_lysc = df.loc[df['Study'].isin(studiesLysC),'Uniprot_final'].unique().tolist()

print(len(df.loc[df['molecular_weight_kDa'].isna(),'Uniprot_final'].unique()))
uniprot_noMW = df.loc[df['molecular_weight_kDa'].isna(),'Uniprot_final'].unique().tolist()


print(len(uniprot_TP),len(uniprot_noMW))


uniprot_all = df['Uniprot_final'].unique().tolist()


#uniprot_toQuery = list(set(uniprot_TP + uniprot_noMW))
uniprot_toQuery = list(set(uniprot_all))

print(len(uniprot_toQuery))
with open('../data/uniprot_toQuery_9May2021.txt', 'w') as f:
    f.write("\n".join(uniprot_toQuery))

with open('../data/uniprot_tryp_9May2021.txt', 'w') as f:
    f.write("\n".join(uniprot_tryp))

with open('../data/uniprot_lysc_9May2021.txt', 'w') as f:
    f.write("\n".join(uniprot_lysc))

with open('../data/uniprot_noMW_9May2021.txt', 'w') as f:
    f.write("\n".join(uniprot_noMW))

#! cut -f1 ../data/uniprot_toQuery.txt | cut -f1 -d"-" | sort | uniq | wc -l
#curl to get Uniprot data
mw_seq = pd.read_csv("../data/main_run",header=None,  sep = '\t')


mw_seq.columns = ['Uniprot_id','MWDa','seq']

mw_seq['MWDa'] = mw_seq['MWDa'].str.replace(",","").astype('float64')
print(len(mw_seq))
mw_seq = mw_seq.drop_duplicates(keep='first').reset_index(drop=True)
print(len(mw_seq))
mw_seq.head()
mw_seq.loc[mw_seq['Uniprot_id']=='P13864']

mw_seq.loc[mw_seq['Uniprot_id']=='Q76I79']

notFoundIds= mw_seq.loc[(mw_seq['MWDa'].isna()) | (mw_seq['seq'].isna()),'Uniprot_id' ].unique()
len(notFoundIds)
len(mw_seq.loc[(mw_seq['MWDa'].isna()) | (mw_seq['seq'].isna())])
len(mw_seq.loc[mw_seq['MWDa'].isna()])
len(mw_seq.loc[mw_seq['seq'].isna()])
mw_seq.loc[(mw_seq['MWDa'].isna()) | (mw_seq['seq'].isna())]
df.loc[df['Uniprot_final'].isin(notFoundIds)].head()
df.loc[df['Uniprot_final'].isin(notFoundIds),'Study'].unique()
add4notFoundIds = df.loc[(df['Uniprot_final'].isin(notFoundIds)) & (df['uniprot_from_gn']=='NoMapping'), 'Uniprot' ].str.split(";").tolist()

add4notFoundIdsUniq = list(set(list(chain.from_iterable(add4notFoundIds))))
len(add4notFoundIdsUniq)
with open('../data/uniprot_toQuery_add4notFoundIdsUniq_9May2021.txt', 'w') as f:
    f.write("\n".join(add4notFoundIdsUniq))
# curl for extra data

mw_seq2 = pd.read_csv("../data/run4notFound",header=None,  sep = '\t')


mw_seq2.columns = ['Uniprot_id','MWDa','seq']

mw_seq2['MWDa'] = mw_seq2['MWDa'].str.replace(",","").astype('float64')
print(len(mw_seq2))
mw_seq2 = mw_seq2.drop_duplicates(keep='first').reset_index(drop=True)
print(len(mw_seq2))
mw_seq2.head()
notFoundIds2 = mw_seq2.loc[(mw_seq2['MWDa'].isna()) | (mw_seq2['seq'].isna()),'Uniprot_id' ].unique()
len(notFoundIds2)

add4notFoundIdsFromGN = list(set(list(chain.from_iterable(df.loc[(df['Uniprot_final'].isin(notFoundIds)) & (df['uniprot_from_gn']!='NoMapping'), 'uniprot_from_gn' ].tolist()))))
 
with open('../data/add4notFoundIdsFromGNUniq_9May2021.txt', 'w') as f:
    f.write("\n".join(add4notFoundIdsFromGN))
#curl extra data

mw_seq3 = pd.read_csv("../data/run4notFoundFromGN",header=None,  sep = '\t')


mw_seq3.columns = ['Uniprot_id','MWDa','seq']

mw_seq3['MWDa'] = mw_seq3['MWDa'].str.replace(",","").astype('float64')
print(len(mw_seq3))
mw_seq3 = mw_seq3.drop_duplicates(keep='first').reset_index(drop=True)
print(len(mw_seq3))
mw_seq3.head()
mw_seq_full = pd.concat([mw_seq, mw_seq2,mw_seq3]).reset_index(drop=True)
mw_seq_full = mw_seq_full.drop_duplicates(keep='first').reset_index(drop=True)
mw_seq_full = mw_seq_full.loc[~((mw_seq_full['MWDa'].isna()) | (mw_seq_full['seq'].isna()))]
len(mw_seq_full)
mw_seq_full = mw_seq.copy()
print(len(mw_seq_full))
mw_seq_full = mw_seq_full.drop_duplicates(keep='first').reset_index(drop=True)
print(len(mw_seq_full))
mw_seq_full = mw_seq_full.loc[~((mw_seq_full['MWDa'].isna()) | (mw_seq_full['seq'].isna()))]
print(len(mw_seq_full))
mw_seq_full['MWkDa'] = mw_seq_full['MWDa']/1000.0
# enzyme-specific cut seq to estimate the number of peptides that are theoretically possible in the experiment

def trypsin_count(seq):
    return len(re.findall('(.{6,29}?(?:R|K|$))', seq)) ### Trypsin

def lysC_count(seq):
    return len(re.findall('(.{6,29}?(?:K|$))', seq)) ### Lys-C

mw_seq_full['trypsin'] = mw_seq_full['seq'].apply(trypsin_count)

mw_seq_full['lysC'] = mw_seq_full['seq'].apply(lysC_count)
mw_seq_full = mw_seq_full.drop(columns=['MWDa','seq'])
mw_seq_full.head()
len(df.loc[(~df['Uniprot_final'].isin(mw_seq_full['Uniprot_id'])) & (df['Study']=='Duda 2018'),'Uniprot_final'].unique())
len(df.loc[(~df['Uniprot_final'].isin(mw_seq_full['Uniprot_id'])) & (df['Study']=='Wisniewski 2015'),'Uniprot_final'].unique())
df.loc[df['Study']=='Wisniewski 2015','molecular_weight_kDa'].unique()
df2 = pd.merge(df, mw_seq_full, left_on = 'Uniprot_final', right_on='Uniprot_id', how='left')
len(df2.loc[(df2['MWkDa'].isna()) | (df2['trypsin'].isna()) | (df2['lysC'].isna())])
df3 = df2.loc[(df2['MWkDa'].isna()) | (df2['trypsin'].isna()) | (df2['lysC'].isna()),['gene_names','gene_name_unified','gene_id_final','Uniprot','Uniprot_unified','uniprot_from_gn','Uniprot_final']].copy().reset_index(drop=True)

def list2str(elem):
    return ";".join(elem)
    
df3['uniprot_from_gn'] = df3['uniprot_from_gn'].apply(list2str)
print(len(df3))
df3 = df3.drop_duplicates(keep='first')
print(len(df3))
print(len(df3.loc[df3['Uniprot_final'].isna()]))

print(len(df3.loc[df3['Uniprot_final']=='NoMapping']))

print(len(df3.loc[df3['Uniprot_final'].str.contains(';')]))
print(len(df3.loc[df3['Uniprot'].isna()]))

print(len(df3.loc[df3['Uniprot']=='NoMapping']))

print(len(df3.loc[df3['Uniprot'].str.contains(';')]))
print(len(df3.loc[df3['uniprot_from_gn'].isna()]))

print(len(df3.loc[df3['uniprot_from_gn']=='NoMapping']))
print(len(df3.loc[df3['uniprot_from_gn']=='N;o;M;a;p;p;i;n;g']))

print(len(df3.loc[df3['uniprot_from_gn'].str.contains(';')]))
mw_seq_full.head()
mw_seq_full_dict = mw_seq_full.set_index('Uniprot_id').T.to_dict('list')
df3.columns
# get MW and theor pep nums 

#df3['currentUniprotID'] = None
df3['MWkDa'] = None
df3['trypsin'] = None
df3['lysC'] = None


df3 = df3.reset_index(drop=True)


for idx,elem in df3.iterrows():
    check = 0
    
    if elem['Uniprot_final'] in mw_seq_full_dict:

        #df3['currentUniprotID'] = i
        df3.loc[idx,'MWkDa'] = mw_seq_full_dict[i][0]
        df3.loc[idx,'trypsin'] = mw_seq_full_dict[i][1]
        df3.loc[idx,'lysC'] = mw_seq_full_dict[i][2]

        check = check+1
            
    else:
        multiMW = list()
        multiTr = list()
        multiLys = list()
        for i in elem['Uniprot'].split(';'):
            i = i.split('-')[0]

            if i in mw_seq_full_dict:
                multiMW.append(mw_seq_full_dict[i][0])
                multiTr.append(mw_seq_full_dict[i][1])
                multiLys.append(mw_seq_full_dict[i][2])
                
                
        if len(multiMW) >0:
            df3.loc[idx,'MWkDa'] = np.median(multiMW)
            df3.loc[idx,'trypsin'] = np.median(multiTr)
            df3.loc[idx,'lysC'] = np.median(multiLys)   

            check = check+1
        
        if ((check == 0) & (elem['uniprot_from_gn'] != 'N;o;M;a;p;p;i;n;g')):
            multiMW = list()
            multiTr = list()
            multiLys = list()

            for j in elem['uniprot_from_gn'].split(';'):
                j = j.split('-')[0]
                if j in mw_seq_full_dict:
                    multiMW.append(mw_seq_full_dict[j][0])
                    multiTr.append(mw_seq_full_dict[j][1])
                    multiLys.append(mw_seq_full_dict[j][2])

            if len(multiMW) >0:
                df3.loc[idx,'MWkDa'] = np.median(multiMW)
                df3.loc[idx,'trypsin'] = np.median(multiTr)
                df3.loc[idx,'lysC'] = np.median(multiLys)          
                    
                check = check+1
                    
                    
        
        elif ((check == 0) & (elem['uniprot_from_gn'] == 'N;o;M;a;p;p;i;n;g')):
            
            print("check ", idx)
            
        else:
            print("ATTENTION",idx)
        
    if check == 0:
        print("elem: ", idx)
            
df3.iloc[30]
len(mw_seq_full_dict)
print(len(df3.loc[df3['MWkDa'].isna()]))
print(len(df3.loc[df3['trypsin'].isna()]))
print(len(df3.loc[df3['lysC'].isna()]))

# these seems to be due to isoforms, where entry is found in uniprot but no MW and seq is returned without specifying isoform 

# isoforms are different, diff MW and diff seq

df3.loc[df3['MWkDa'].isna()].head()
len(df3.loc[df3['MWkDa'].isna(),'gene_id_final'].unique())
len(df3.loc[(df3['MWkDa'].isna()) & (df3['Uniprot'].str.contains('-')),'gene_id_final'].unique()) # only for 1 gene possible prot isoform is known
len(df2.loc[df2['gene_id_final'].isin(df3.loc[df3['MWkDa'].isna(),'gene_id_final'].unique())])/len(df2) # 0.05% of genes -> drop is to be more accurate
df2nn = df2.loc[(df2['MWkDa'].isna()) | (df2['trypsin'].isna()) | (df2['lysC'].isna())].copy()
print(len(df2))

df2 = df2.loc[~( (df2['MWkDa'].isna()) | (df2['trypsin'].isna()) | (df2['lysC'].isna()) )]

print(len(df2))
print(len(df2nn))
df2nn = df2nn.drop(columns=['Uniprot_id', 'MWkDa', 'trypsin','lysC'])
df2nn.columns
df3.columns
df3 = df3.drop(columns=['uniprot_from_gn'])
df2nnm = pd.merge(df2nn, df3, how='inner', on = ['gene_names', 'gene_name_unified','gene_id_final', 'Uniprot', 'Uniprot_unified','Uniprot_final'])
len(df2nnm)
df2 = df2.drop(columns='Uniprot_id')


df4conc = pd.concat([df2,df2nnm], ignore_index=True,  sort=False)
df4conc.loc[df4conc['Study'].isin(studiesTrypsin),'TheorPepNum'] = df4conc.loc[df4conc['Study'].isin(studiesTrypsin),'trypsin']

df4conc.loc[df4conc['Study'].isin(studiesLysC),'TheorPepNum'] = df4conc.loc[df4conc['Study'].isin(studiesLysC),'lysC']

len(df4conc.loc[df4conc['molecular_weight_kDa'].isna()])
df4conc.loc[df4conc['molecular_weight_kDa'].isna(),'molecular_weight_kDa'] = df4conc.loc[df4conc['molecular_weight_kDa'].isna(),'MWkDa']
len(df4conc.loc[df4conc['molecular_weight_kDa'].isna()])
df4conc.loc[df4conc['molecular_weight_kDa'] != df4conc['MWkDa'],['molecular_weight_kDa','MWkDa']].drop_duplicates(keep='first').head()
df4conc = df4conc.drop(columns = ['MWkDa', 'trypsin', 'lysC'])
df4conc = df4conc.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Calculate concentrations  based on total protein approach from Wiśniewski et al. https://doi.org/10.1074/mcp.M113.037309<jupyter_code>df = df4conc.copy()
df['conc_uM'] = None
df['log_conc_uM'] = None

df['copyNum'] = None
df['totalProtein'] = None
df['totalVolume'] = None<jupyter_output><empty_output><jupyter_text>Median cellular concentration [nM]<jupyter_code>df.loc[df['raw_data_units']=='Median cellular concentration [nM]','Study'].unique()
df.loc[df['raw_data_units']=='Median cellular concentration [nM]','conc_uM'] = df.loc[df['raw_data_units']=='Median cellular concentration [nM]','raw_data']/1000.0 # nM to uM<jupyter_output><empty_output><jupyter_text>LFQintensity<jupyter_code>df.loc[df['TheorPepNum']==np.max(df['TheorPepNum']),'gene_id_final'].unique() # TTN = Titin
df['sample_full_id'] = df['Study'].astype(str) + '_' + df['Organism'].astype(str) + '_' + df['location'].astype(str)  + '_' + df['Age_days'].astype(str) + '_' + df['condition'].astype(str)  + '_' + df['sample_id'].astype(str) 

len(df['sample_full_id'].unique())
len(df.loc[df['raw_data_units']=='LFQintensity','sample_full_id'].unique())
def protRulLFQ(lfq,detectabilityNormFactor, mw_kDa):
    tcp = 200 #tcp = Total cellular protein concentration [g/l] is 200-300 g/l typically 
    papc = 200 # "Protein amount per cell [pg]" =  200 
    avogadro = 6.02214129e23
    
    mwWeightedNormalizedSummedIntensities = np.sum(lfq/detectabilityNormFactor*(mw_kDa*1000))
    factor = papc*1e-12*avogadro/mwWeightedNormalizedSummedIntensities
    
    copyNum = lfq/detectabilityNormFactor*factor # if detectability is the number of theorretical peptides  
    totalProtein = np.sum(copyNum*mw_kDa*1000*1e12/avogadro) # *1000 because of kDa to Da
    totalVolume = totalProtein/tcp*1000 # femtoliters
    
    calcConc = copyNum/(totalVolume*1e-15)/avogadro*1e6 # micromolar
    
    return([calcConc,copyNum,totalProtein,totalVolume])
# this will take a while

for sample_full_id in df.loc[df['raw_data_units']=='LFQintensity','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRulLFQ(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'TheorPepNum'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])<jupyter_output>Hamezah 2019_mouse_hippocampus_476_WT_nan
Hamezah 2019_mouse_hippocampus_476_Alzheimer_nan
Hamezah 2019_mouse_striatum_476_WT_nan
Hamezah 2019_mouse_striatum_476_Alzheimer_nan
Hamezah 2019_mouse_cortex_476_WT_nan
Hamezah 2019_mouse_cortex_476_Alzheimer_nan
Krogager 2018_mouse_striatum_112_control_nan
Krogager 2018_mouse_neurons_112_SORT_nan
Sharma 2015, cultured_mouse_microglia_0_nan_adultMicroglia1
Sharma 2015, cultured_mouse_microglia_0_nan_adultMicroglia2
Sharma 2015, cultured_mouse_microglia_0_nan_adultMicroglia3
Sharma 2015, cultured_mouse_microglia_0_nan_youngMicroglia1
Sharma 2015, cultured_mouse_microglia_0_nan_youngMicroglia2
Sharma 2015, cultured_mouse_microglia_0_nan_youngMicroglia3
Sharma 2015, cultured_mouse_astrocytes_0_nan_Astrocytes1
Sharma 2015, cultured_mouse_astrocytes_0_nan_Astrocytes2
Sharma 2015, cultured_mouse_astrocytes_0_nan_Astrocytes3
Sharma 2015, cultured_mouse_neurons_0_nan_Neuronsdiv051
Sharma 2015, cultured_mouse_neurons_0_nan_Neuronsdiv052
Sharma 2015, c[...]<jupyter_text>iBAQ<jupyter_code>#iBAQ: total intensity divided by the number of tryptic peptides between 6 and 30 amino acids in length. <NAME> et al 2013
def protRulIBAQ(ibaq, mw_kDa):
    tcp = 200 #tcp = Total cellular protein concentration [g/l] is 200-300 g/l typically 
    papc = 200 # "Protein amount per cell [pg]" =  200 
    avogadro = 6.02214129e23
    
    mwWeightedNormalizedSummedIntensities = np.sum(ibaq*(mw_kDa*1000))
    factor = papc*1e-12*avogadro/mwWeightedNormalizedSummedIntensities
    
    copyNum = ibaq*factor  # adapted for iBAQ as sharma['detectabilityNormFactor'] = sharma['MedPepCount'] 
    
    totalProtein = np.sum(copyNum*mw_kDa*1000*1e12/avogadro) # *1000 because of kDa to Da
    totalVolume = totalProtein/tcp*1000 # femtoliters
    calcConc = copyNum/(totalVolume*1e-15)/avogadro*1e6 # micromolar
    
    return([calcConc,copyNum,totalProtein,totalVolume])<jupyter_output><empty_output><jupyter_text>df.loc[df['raw_data_units']=='iBAQ','conc_uM'] = protRulIBAQ(df.loc[df['raw_data_units']=='iBAQ','raw_data'],                                                                    df.loc[df['raw_data_units']=='iBAQ','molecular_weight_kDa'])<jupyter_code>for sample_full_id in df.loc[df['raw_data_units']=='iBAQ','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRulIBAQ(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])<jupyter_output><empty_output><jupyter_text>Protein concentration (mol/g protein)<jupyter_code>def protRul_molPerGramProt(molgramprot, mw_kDa):
    tcp = 200 #tcp = Total cellular protein concentration [g/l] is 200-300 g/l typically 
    papc = 200 # "Protein amount per cell [pg]" =  200 
    avogadro = 6.02214129e23
    
    copyNum = molgramprot*avogadro*papc*1e-12  
    
    totalProtein = np.sum(copyNum*mw_kDa*1000*1e12/avogadro) # *1000 because of kDa to Da
    totalVolume = totalProtein/tcp*1000 # femtoliters
    calcConc = copyNum/(totalVolume*1e-15)/avogadro*1e6 # micromolar
    
    return([calcConc,copyNum,totalProtein,totalVolume])
for sample_full_id in df.loc[df['raw_data_units']=='Protein concentration (mol/g protein)','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRul_molPerGramProt(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])  
    
len(df.loc[(df['Study']=='Wisniewski 2015') & (df['molecular_weight_kDa'].isna())])<jupyter_output><empty_output><jupyter_text>Mean concentration [mol/(g total protein)]<jupyter_code># rat brain density: 1.04-1.05 g/ml https://link.springer.com/chapter/10.1007/978-3-7091-9115-6_12 https://www.researchgate.net/post/What_is_the_average_wet_weight_for_dissected_RAT_brain_areas_substantia_nigra_striatum_septum_hippocampus_nucleus_accumbens_or_VTA
# human brain density: 1.03 kg/l

# From Fischer et al 2004 Average protein density is a molecular-weight-dependent function:
# ro = 1.41 + 0.145 * exp(-M(kDa)/13(4)) g/cm3 1cm3 = 0.001L = 1 mL
# or ~1.4 g/cm3 = 1.4 g/ml = 1.4*1000.0 g/l

# 1 cm3 = 1e-3 L = 1 ml


def protRul_molPerGramProt(molgramprot, mw_kDa):
    tcp = 200 #tcp = Total cellular protein concentration [g/l] is 200-300 g/l typically 
    papc = 200 # "Protein amount per cell [pg]" =  200 
    avogadro = 6.02214129e23
    
    copyNum = molgramprot*avogadro*papc*1e-12  
    
    totalProtein = np.sum(copyNum*mw_kDa*1000*1e12/avogadro) # *1000 because of kDa to Da
    totalVolume = totalProtein/tcp*1000 # femtoliters
    calcConc = copyNum/(totalVolume*1e-15)/avogadro*1e6 # micromolar
    
    return([calcConc,copyNum,totalProtein,totalVolume])
for sample_full_id in df.loc[df['raw_data_units']=='Mean concentration [mol/(g total protein)]','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRul_molPerGramProt(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])<jupyter_output><empty_output><jupyter_text>IntensityL<jupyter_code># SILAC data, apply the same method as in LFQ, approximation

for sample_full_id in df.loc[df['raw_data_units']=='IntensityL','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRulLFQ(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'TheorPepNum'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])<jupyter_output>Geiger 2013_mouse_cortex_91_nan_nan
Geiger 2013_mouse_medulla_91_nan_nan
Geiger 2013_mouse_cerebellum_91_nan_nan
Geiger 2013_mouse_midbrain_91_nan_nan
Geiger 2013_mouse_olfactory bulb_91_nan_nan<jupyter_text>tmt abundance<jupyter_code>#apply the same method as in LFQ, approximation

for sample_full_id in df.loc[df['raw_data_units']=='tmt abundance','sample_full_id'].unique():
    #print(sample_full_id)
    df.loc[df['sample_full_id']==sample_full_id,['conc_uM','copyNum','totalProtein','totalVolume']] = protRulLFQ(df.loc[df['sample_full_id']==sample_full_id,'raw_data'],df.loc[df['sample_full_id']==sample_full_id,'TheorPepNum'],df.loc[df['sample_full_id']==sample_full_id,'molecular_weight_kDa'])
    
df['log_conc_uM'] = np.log(df['conc_uM'].astype('float64'))

df['log_conc_uM'] = df['log_conc_uM'].astype('float64')
df = df.reset_index(drop=True)
df.columns

fig = plt.figure(figsize=(12, 6))


ax = sns.violinplot(x="Study", y="log_conc_uM", data=df)

ax.grid(False)
plt.xticks(rotation=75)
ax.set_ylabel('log_conc_uM')

plt.show()
with open('../data/3_df_with_conc_PerSampleNorm_9May2021.pkl','wb') as f:
    pkl.dump(df,f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17746196370040604, 'fraction_numerical': 0.03781615380852209, 'mean_word_length': 6.149877579573277, 'pattern_counts': {'":': 1, '<': 68, '<?xml version=': 0, '>': 79, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '30369', 'license': 'Apache-2.0', 'n_tokens_mistral': 18212, 'n_tokens_neox': 16402, 'n_words': 2156, 'path': 'step_3_protein_conc_calc_tpaPerSample.ipynb', 'repo_name': 'BlueBrain/MADIP', 'size': '40891'}