<jupyter_start><jupyter_text>This file is part of MADIP: Molecular Atlas Data Integration PipelineThis file cleans metadataCopyright 2021 Blue Brain Project / EPFL    Licensed under the Apache License, Version 2.0 (the "License");   you may not use this file except in compliance with the License.   You may obtain a copy of the License at       http://www.apache.org/licenses/LICENSE-2.0   Unless required by applicable law or agreed to in writing, software   distributed under the License is distributed on an "AS IS" BASIS,   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   See the License for the specific language governing permissions and   limitations under the License.<jupyter_code>import pandas as pd
import numpy as np

from matplotlib import pyplot as plt
import seaborn as sns

import re

import pickle as pkl

import scipy.stats as st

from matplotlib.cbook import boxplot_stats

import timeit
from collections import Counter

from numpy.random import seed
from numpy.random import randint
from scipy.stats import ks_2samp
from scipy.stats import levene
from scipy.stats import kruskal

import scikit_posthocs as sp
pd.options.display.max_columns = None
pd.options.display.max_rows = None

%matplotlib inline
%config InlineBackend.figure_format = 'retina'



sns.set_style('whitegrid',{'axes.grid':False})
with open('../data/3_df_with_conc_PerSampleNorm.pkl','rb') as f:
    df = pkl.load(f)
print(len(df))

df.loc[df['gene_id_final'].str.contains(';')]
df.columns<jupyter_output><empty_output><jupyter_text>Clean metadata<jupyter_code>df.loc[df['Organism'].isna(),'Study'].unique()
df.loc[df['location'].isna(),'Study'].unique()
df.loc[df['Age_days'].isna(),'Study'].unique()
df.loc[df['Age_cat'].isna(),'Study'].unique()
df['Age_cat'].unique()
# Age_cat for mouse (C57BL/6J):

# embr: before birth
# infants/young: from birth to 21 days of age 
# juvenile/young: from 3 weeks to 8 weeks old
# adults: after 2 months (here consider inkl. according to data sources papers)

# mature adult: 3-6 months
# middle-aged: 10-14 months
# old: 18-24 months

# https://www.jax.org/news-and-insights/jax-blog/2017/november/when-are-mice-considered-old# https://www.researchgate.net/post/At_what_age_are_laboratory_mice_considered_adult2

# or according to the sources where explicitly stated
df.loc[~df['Age_cat'].isna(),'Study'].unique()
print(df.loc[df['Study']=='Hamezah 2019','Age_days'].unique()) # 15*30 #Five-month-old mice were divided into four groups ... for a duration of 10 months -> 15 months

df.loc[df['Study']=='Hamezah 2019','Age_cat'] = 'middle-aged' # https://www.jax.org/news-and-insights/jax-blog/2017/november/when-are-mice-considered-old#

print(df.loc[df['Study']=='Sharma 2015, isolated','Age_days'].unique())


df.loc[(df['Study']=='Sharma 2015, isolated')&(df['Age_days']==29.0),'Age_cat'] = 'young' # 29 #8 + 21 # 'cultured cells? or isolated?' 'P8'

df.loc[(df['Study']=='Sharma 2015, isolated')&(df['Age_days']==81.0),'Age_cat'] ='adult' # 81 # 60 + 21 'P60'

df.loc[(df['Study']=='Sharma 2015, isolated') &(df['Age_days']==26.0),'Age_cat'] = 'young' # 26 #5 + 21 # 'cultured cells? or isolated?' 'P5'
df.loc[(df['Study']=='Sharma 2015, isolated') &(df['Age_days']==35.0),'Age_cat'] = 'young' # 35 #14 + 21 # 'cultured cells? or isolated?' 'P14'
df.loc[(df['Study']=='Sharma 2015, isolated') &(df['Age_days']==45.0),'Age_cat'] = 'young' # 45 #24 + 21 # 'cultured cells? or isolated?' 'P24'

print(df.loc[df['Study']=='Sharma 2015, cultured','Age_days'].unique())

df.loc[df['Study']=='Sharma 2015, cultured','Age_cat'] = 'embr'
print(df.loc[df['Study']=='Hosp 2017, soluble','Age_days'].unique())

df.loc[(df['Study']=='Hosp 2017, soluble') & (df['Age_days']==56.0),'Age_cat'] = 'young'
df.loc[(df['Study']=='Hosp 2017, soluble') & (df['Age_days']==77.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Hosp 2017, soluble') & (df['Age_days']==105.0),'Age_cat'] = 'adult'
print(df.loc[df['Study']=='Hosp 2017, insoluble','Age_days'].unique())

df.loc[(df['Study']=='Hosp 2017, insoluble') & (df['Age_days']==56.0),'Age_cat'] = 'young'
df.loc[(df['Study']=='Hosp 2017, insoluble') & (df['Age_days']==77.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Hosp 2017, insoluble') & (df['Age_days']==105.0),'Age_cat'] = 'adult'
print(df.loc[df['Study']=='Hosp 2017, CSF','Age_days'].unique())

df.loc[(df['Study']=='Hosp 2017, CSF') & (df['Age_days']==56.0),'Age_cat'] = 'young'
df.loc[(df['Study']=='Hosp 2017, CSF') & (df['Age_days']==77.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Hosp 2017, CSF') & (df['Age_days']==105.0),'Age_cat'] = 'adult'
print(df.loc[df['Study']=='Geiger 2013','Age_days'].unique())

df.loc[df['Study']=='Geiger 2013','Age_cat'] = 'adult'
print(df.loc[df['Study']=='Wisniewski 2015','Age_days'].unique())

df.loc[df['Study']=='Wisniewski 2015','Age_cat'] = 'adult'
print(df.loc[df['Study']=='Han 2014','Age_days'].unique())

df.loc[df['Study']=='Han 2014','Age_cat'] = 'embr'
print(df.loc[df['Study']=='Kjell 2020','Age_days'].unique())

df.loc[df['Study']=='Kjell 2020','Age_cat'] = 'adult'
print(df.loc[df['Study']=='Krogager 2018','Age_days'].unique())

df.loc[df['Study']=='Krogager 2018','Age_cat'] = 'adult'
print(df.loc[df['Study']=='Hamezah 2018','Age_days'].unique())

#Hamezah - rats, lifespan differ from mouse
# rat lifespan https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3733029/ making correspondence to mouse
df.loc[(df['Study']=='Hamezah 2018') &(df['Age_days']==446.0),'Age_cat'] = 'adult' #446 # 365 + 2*30 + 21 # '14 months'
df.loc[(df['Study']=='Hamezah 2018') &(df['Age_days']==566.0),'Age_cat'] = 'mature adult' #569 # 365 + 365/2 +21 # '18 months'
df.loc[(df['Study']=='Hamezah 2018') &(df['Age_days']==721.0),'Age_cat'] = 'middle-aged' #721 # 365*2 -30 +21 # '23 months'
df.loc[(df['Study']=='Hamezah 2018') &(df['Age_days']==841.0),'Age_cat'] = 'old' #841 # 365*2 + 30*3 +21 # '27 months'


print(df.loc[df['Study']=='Duda 2018','Age_days'].unique())

df.loc[(df['Study']=='Duda 2018') & (df['Age_days']==51.0),'Age_cat'] = 'young'
df.loc[(df['Study']=='Duda 2018') & (df['Age_days']==386.0),'Age_cat'] = 'middle-aged'
print(df.loc[df['Study']=='Carlyle 2017','Age_days'].unique())

#human

df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==386.0),'Age_cat'] = 'child'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==631.0),'Age_cat'] = 'child'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==996.0),'Age_cat'] = 'child'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==1361.0),'Age_cat'] = 'child' 
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==1726.0),'Age_cat'] = 'child' 
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==3186.0),'Age_cat'] = 'child'  
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==4281.0),'Age_cat'] = 'adolescence'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==5741.0),'Age_cat'] = 'adolescence'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==7201.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==8661.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==11216.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==13406.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']== 13771.0),'Age_cat'] = 'adult'
df.loc[(df['Study']=='Carlyle 2017')& (df['Age_days']==14866.0),'Age_cat'] = 'adult'
print(df.loc[df['Study']=='Zhu 2018','Age_days'].unique())

# rat P17

df.loc[df['Study']=='Zhu 2018','Age_cat'] = 'young'
print(df.loc[df['Study']=='Fecher 2019','Age_days'].unique()) # adult 

df.loc[df['Study']=='Fecher 2019','Age_cat'] = 'adult'
print(df.loc[df['Study']=='Bai 2020','Age_days'].unique())

df.loc[(df['Study']=='Bai 2020') & (df['Age_days']==111.0),'Age_cat'] = 'adult' #3months
df.loc[(df['Study']=='Bai 2020') & (df['Age_days']==201.0),'Age_cat'] = 'mature adult' #6months
df.loc[(df['Study']=='Bai 2020') & (df['Age_days']==386.0),'Age_cat'] = 'middle-aged' #12months
df.loc[(df['Study']=='Bai 2020') & (df['Age_days']=='post-mortem'),'Age_cat'] =  'post-mortem' #
print(df.loc[df['Study']=='Hasan 2020','Age_days'].unique())

df.loc[df['Study']=='Hasan 2020','Age_cat'] = 'mature adult'
df.loc[df['condition']=='AD_severe','Study'].unique()
df['condition'].unique()
# do not replace conditon names as may lead to duplicates


#df.loc[(df['condition'].isin(['WT','control', 'SORT','young', 'adult']) | (df['condition'].isna())),'condition' ] = "control"

#df.loc[df['condition'].isin(['Alzheimer','AD']),'condition' ] = "Alzheimer"

#df.loc[df['condition']=='EAE','condition' ] = "experimental autoimmune encephalomyelitis" # too long name for plots
with open('../data/4_df_agecat.pkl','wb') as f:
    pkl.dump(df,f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24637184321791794, 'fraction_numerical': 0.06444977716832362, 'mean_word_length': 7.1187384044526905, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '125854', 'license': 'Apache-2.0', 'n_tokens_mistral': 4140, 'n_tokens_neox': 3336, 'n_words': 608, 'path': 'step_4_cleanMetadata.ipynb', 'repo_name': 'BlueBrain/MADIP', 'size': '8751'}