<jupyter_start><jupyter_text>Overview of this notebook:1. convert to two class (c-/abshort &c+/ablong) and the holdout c-/ablong2. split the train and test set3. median standardize4. aggregate5. imputei do 1-3 BEFORE I AGGREGATE and median standardize, as some information may be leaking from the train/test set as i had it previously.<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import os
from pathlib import Path
import seaborn as sns
import numpy as np
import glob

from sklearn.externals.joblib import Memory
from sklearn.model_selection import train_test_split


%matplotlib inline
plt.style.use('ggplot')

from notebook.services.config import ConfigManager
cm = ConfigManager()
cm.update('livereveal', {
        'width': 1024,
        'height': 768,
        'scroll': True,
})

%load_ext autotime
#patients of interest from rotation_cohort_generation
from parameters import final_pt_df_v, date, repository_path, categorical

#patients of interest from rotation_cohort_generation
final_pt_df2 = final_pt_df_v #pd.read_csv('/Users/geickelb1/Documents/GitHub/mimiciii-antibiotics-modeling/data/raw/csv/%s_final_pt_df2.csv'%(most_updated_patient_df), index_col=0)
del(final_pt_df_v)

patients= list(final_pt_df2['subject_id'].unique())
hadm_id= list(final_pt_df2['hadm_id'].unique())
icustay_id= list(final_pt_df2['icustay_id'].unique())
icustay_id= [int(x) for x in icustay_id]

len(patients)

def save_df(df, df_name='default', add_subfolder=False):
    #uses the date and supplied df name and saves to the savepath specified above.
    
    save_path= str(repository_path)+'/data/cleaned_merged_agg/'
    if df_name == 'default':
        df_name= "%s"%(df)
    
    address=save_path+'%s/'%(folder)
    if not os.path.exists(address):
        print(address)
        os.makedirs(address)
    pd.DataFrame(df).to_csv(Path(address+'%s_%s_cleaned_merged_agg.csv' %(date, df_name)))
from parameters import lower_window, upper_window, folder, date, time_col, time_var, patient_df

#importing cleaned_merged big_df
allFiles = glob.glob(str(repository_path)+ '/data/cleaned_merged/%s/'%(folder) + "{}_*.csv".format(date))

df_list=[]
for element in allFiles:
    df_list.append(element.split('{}_'.format(date))[1].split('_prepped.csv')[0]) #making an list of all my dataframes in order they appear in file

i=0
for name in df_list:
    big_df = pd.read_csv(allFiles[i],  index_col=0)
#quick housekeeping addition to accomidate older generated data
if len(big_df.loc[big_df['label']=="pao2/fio2",'label'])>1:
    big_df.loc[big_df['label']=="pao2/fio2",'label']="pao2fio2ratio"
big_df.head()
#overview of all variables and formats
big_df.groupby('label')['value'].describe().sort_values('unique')<jupyter_output><empty_output><jupyter_text>initial data prep* convert to two class<jupyter_code>#convert to two class
final_pt_df2['final_bin'].unique()
two_classes=['C_neg/A_partial','C_pos/A_full']
two_class_icu=final_pt_df2.loc[final_pt_df2.loc[:,"final_bin"].isin(two_classes),['icustay_id','subject_id','final_bin']]

# hold out class
ho_class=['C_neg/A_full']
one_class_icu=final_pt_df2.loc[final_pt_df2.loc[:,"final_bin"].isin(ho_class),['icustay_id','subject_id','final_bin']]

big_df_ho=big_df.loc[big_df['icustay_id'].isin(list(one_class_icu['icustay_id'])),:].copy()



big_df=big_df.loc[big_df['icustay_id'].isin(list(two_class_icu['icustay_id'])),:].copy()
len(big_df)
big_df['icustay_id'].nunique()<jupyter_output><empty_output><jupyter_text>roundabout way of sampling train/test set so that each subject is only in either train or test:* To ensure a single patient did not end up with samples in both training and testing sets, individual patients (subject_id's) are kept together when performing the stratified train and test split<jupyter_code># label each subject_id with the max of the two classes. 
two_class_pt=two_class_icu.copy()
two_class_pt['final_bin']=pd.factorize(two_class_pt['final_bin'])[0]
two_class_maxsub=two_class_pt.loc[two_class_pt.groupby('subject_id')['final_bin'].idxmax(),:]
# 70/30 train/test set split with 12345=seed, splitting on max final bin of each SUBJECT_ID
train, test = train_test_split(two_class_maxsub, test_size=0.3, random_state=12345, 
                               stratify=two_class_maxsub['final_bin'])
# generate list of each SUBJECT_ID in each split
train_subject=list(train['subject_id'])
test_subject=list(test['subject_id'])

#filtering big_df on train subjects and test subjects to get my train/test splits.
big_df_train= big_df.loc[big_df.loc[:,'subject_id'].isin(train_subject),:].copy()
big_df_test= big_df.loc[big_df.loc[:,'subject_id'].isin(test_subject),:].copy()
del big_df<jupyter_output>time: 723 ms<jupyter_text>calc median/iqr of 'healthiest patients' for standardizationtake all non-categorical variables for HEALTHY PATIENTS and calculate the median and IQR for them. then will use this to make z scores via: $$Z=\frac{(X-\widetilde{X}_{-/short})}{(IQR_{-/short})}$$ where $\widetilde{X}_{-/short}$ is the median value of the patients with negative SSC and short duration EAT.<jupyter_code>big_df_train[big_df_train['label']=='vent_recieved']['icustay_id'].nunique()
big_df_train['icustay_id'].nunique()
from parameters import continuous, onetime

def median_label_fxn(big_df):
    """ grabs the median and IQR values for all continuous variables based upon the values present in
    the last 24 hours of the c_neg/A_partial patients (AKA the healthiest patients at the healthiest timepoints) """
    
    global final_pt_df2
    healthy_pt=list(final_pt_df2[final_pt_df2['final_bin']=="C_neg/A_partial"]['icustay_id'])

    #filter to only healthy patients filter
    ##splitting big_df, making a copy and restricting it to all values that will be usd in standardizing
    healthy_df=big_df[big_df['label'].isin(continuous+onetime)].copy()
    healthy_df['value']= pd.to_numeric(healthy_df['value']) #converting to numeric 
    healthy_df=healthy_df[healthy_df['icustay_id'].isin(healthy_pt)].copy() #only numerical values for cneg/ab partial pt
    
    #finding the last 24 hours of each healthy patient.
    healthy_pt_end=pd.DataFrame(healthy_df.groupby("icustay_id")['delta'].max())
    healthy_pt_end["start"]=healthy_pt_end['delta']- pd.to_timedelta("1 day 00:00:00")
    healthy_pt_end=healthy_pt_end.rename(columns={"delta":"end"}).reset_index()
    healthy_df= pd.merge(healthy_df,healthy_pt_end, left_on="icustay_id", right_on="icustay_id", how="left" ) #now have the last 24 hours annotated for each patient as start and end.
    
    #calculating medians and iqr for each label based on healthy patient's last 24 hours in icu
    median_label=pd.DataFrame((healthy_df.groupby("label")['value'].median())).reset_index()
    median_label=median_label.rename(columns={'value':"median"})
    iqr_label=pd.DataFrame((healthy_df.groupby("label")['value'].quantile(0.75)-healthy_df.groupby("label")['value'].quantile(0.25))).reset_index()
    iqr_label=iqr_label.rename(columns={'value':"iqr"})
    median_label=pd.merge(median_label,iqr_label)  #final median df
    return(median_label)

def standardization_fxn(big_df, big_df_2class=None): ##changed here
    global continuous, onetime, vaso_active, ordinal, categorical
    
    if big_df_2class is not None:
        median_label=median_label_fxn(big_df_2class)
    else:
        median_label=median_label_fxn(big_df)
    
    ### dataformatting: convert all dtypes to a numeric type that pereserves nan. 
    #splitting categorical, ordinal and continuous
    big_categorical= big_df.loc[big_df.loc[:,'label'].isin(categorical),:].copy() 

    #continuous and ordinal variables
    big_noCat= big_df.loc[big_df.loc[:,'label'].isin(continuous+onetime),:].copy() 
    big_noCat['value']= big_noCat['value'].apply(pd.to_numeric, args=('coerce',)) #instead of convert to float, may preserve nan's better. 
    
    ### adding a standardized value (x-median)/iqr  where median is of the last 24 hours in time window for culture neg/ ab partial patients 
    big_noCat=pd.merge(big_noCat, median_label, how="left") 
    #loging values
    big_noCat['median']= np.log(big_noCat['median']+1.0)
    big_noCat['iqr']= np.log(big_noCat['iqr']+1.0)
    big_noCat['raw_value']=big_noCat['value']
    big_noCat['value']=np.log(big_noCat['value']+1.0)
    
    big_noCat['standardize']=((big_noCat['value']-big_noCat['median'])/big_noCat['iqr']).fillna(0) #standardize is log standardized  

    return(big_noCat, big_categorical)
noCat_df_train, cat_df_train = standardization_fxn(big_df_train)
noCat_df_test, cat_df_test = standardization_fxn(big_df_test)
##### holdout
noCat_df_ho, cat_df_ho = standardization_fxn(big_df_ho, big_df_train)

#should be same number of col as len(categorical)
cat_df_train[cat_df_train['icustay_id']==200012.0].groupby('label')['value'].value_counts()#.head(10)
noCat_df_ho[noCat_df_ho['label']=='weight'].head()
noCat_df_ho[noCat_df_ho['label']=='lactate'].head()
noCat_df_train[noCat_df_train['label']=='lactate']<jupyter_output><empty_output><jupyter_text>aggregation: clincally guided min/max<jupyter_code>from parameters import low_value, hi_value, both_value

# important_onetime=['yearsold','weight'] #pco2 and bands are now in categorical 12-12-19

noCat_df_train['label'].unique()#noCat_df_train['label']=='any_vasoactives'<jupyter_output><empty_output><jupyter_text>running the min/max aggregations.<jupyter_code>def clin_agg(big_noCat, big_cat, big_df, values="standardize"):
    """
    clincally guided aggregations.
    
    values= choose here if want to use standardization or raw values.
    
    note: getting two minor errors, could use some cleaning up at later date.
    """
    global hi_value, low_value, both_value, important_onetime
    #max aggregation for selected variables
    big_max= big_noCat.loc[big_noCat.loc[:,'label'].isin(hi_value),:]
    table = pd.pivot_table(big_max, values=values, columns='label', index=['icustay_id'],aggfunc=max, dropna=False)
    
    #min aggregation for selected variables
    big_min= big_noCat.loc[big_noCat.loc[:,'label'].isin(low_value),:]
    table2 = pd.pivot_table(big_min, values=values, columns='label', index=['icustay_id'],aggfunc=min, dropna=False)
    
    #max&min aggregation for selected variables
    big_both= big_noCat.loc[big_noCat.loc[:,'label'].isin(both_value),:]
    table3 = pd.pivot_table(big_both, values=values, columns='label', index=['icustay_id'],aggfunc=[max,min], dropna=False)
    
    #first left join all different continuous aggregations together. 
    worst_df=pd.merge(table.reset_index(), table2.reset_index(), how='left')
    worst_df=pd.merge(worst_df, table3.reset_index(), left_on='icustay_id', right_on='icustay_id',how='left')
    
    ### formatting categorical to wide format to match the tables/worst_df
    big_cat= big_cat.pivot(
    index='icustay_id',
    values='value',
    columns='label').reset_index() #need to convert to wide format. should be one row per icustay per time. 
    
    ## merging the categorical and aggregated dataframes together. 
    worst_df=pd.merge(worst_df, big_cat, left_on='icustay_id', right_on='icustay_id',how='left') #using max/min aggregates 

#     #adding important one_time values to final aggregated
#     agg_remaining= big_df.loc[big_df.loc[:,'label'].isin(important_onetime),:]
#     agg_table2 = pd.pivot_table(agg_remaining, values='value', columns='label', index=['icustay_id'],aggfunc=[max], dropna=False) 
#     agg_table2.columns = agg_table2.columns.get_level_values(1)
#     agg_table2=agg_table2.reset_index()
#     agg_table2.head()#.rename(columns={})

#     worst_df=pd.merge(worst_df, agg_table2, how='left')
    return(worst_df)

hi_value
worst_df_train=clin_agg(noCat_df_train, cat_df_train,big_df_train, values="standardize")
worst_df_test=clin_agg(noCat_df_test, cat_df_test, big_df_test, values="standardize")
worst_df_ho=clin_agg(noCat_df_ho, cat_df_ho,big_df_ho, values="standardize")
worst_df_train.head()
worst_df_ho['pao2fio2ratio'].describe()
worst_df_ho['weight'].describe()
worst_df_train['pao2fio2ratio'].describe()
worst_df_train['vasopressin'].value_counts()
#worst_df_train['vasopressin'].value_counts()
worst_df_train['bands'].value_counts()
worst_df_train['pco2'].value_counts()
save_df(worst_df_train, 'train')
save_df(worst_df_test, 'test')
save_df(worst_df_ho, 'longneg')
del worst_df_train, worst_df_test<jupyter_output>time: 6 ms
{'fraction_non_alphanumeric': 0.13138156302927656, 'fraction_numerical': 0.010242761513025243, 'mean_word_length': 6.311320754716981, 'pattern_counts': {'":': 1, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '145663', 'license': 'MIT', 'n_tokens_mistral': 4543, 'n_tokens_neox': 4342, 'n_words': 869, 'path': 'notebooks/outcomes_and_alternative_calcs/06-feature_aggregation-long_neg.ipynb', 'repo_name': 'geickelb/mimiciii-antibiotics-opensource', 'size': '12399'}