<jupyter_start><jupyter_text>initial data prep<jupyter_code>#convert to two class

final_pt_df2['final_bin'].unique()
two_classes=['C_neg/A_partial','C_pos/A_full']
two_class_icu=final_pt_df2.loc[final_pt_df2.loc[:,"final_bin"].isin(two_classes),['icustay_id','subject_id','final_bin']]
big_df=big_df.loc[big_df['icustay_id'].isin(list(two_class_icu['icustay_id'])),:].copy()
len(big_df)#7015776 -> ~3000000
big_df['icustay_id'].nunique() #14181 ->7588<jupyter_output><empty_output><jupyter_text>roundabout way of sampling train/test set so that each subject is only in either train or test:<jupyter_code># label each subject_id with the max of the two classes. 
two_class_pt=two_class_icu.copy()
two_class_pt['final_bin']=pd.factorize(two_class_pt['final_bin'])[0]
two_class_maxsub=two_class_pt.loc[two_class_pt.groupby('subject_id')['final_bin'].idxmax(),:]
# 70/30 train/test set split with 12345=seed, splitting on max final bin of each SUBJECT_ID
train, test = train_test_split(two_class_maxsub, test_size=0.3, random_state=12345, 
                               stratify=two_class_maxsub['final_bin'])
# generate list of each SUBJECT_ID in each split
train_subject=list(train['subject_id'])
test_subject=list(test['subject_id'])

#filtering big_df on train subjects and test subjects to get my train/test splits.
big_df_train= big_df.loc[big_df.loc[:,'subject_id'].isin(train_subject),:].copy()
big_df_test= big_df.loc[big_df.loc[:,'subject_id'].isin(test_subject),:].copy()
del big_df
# #converting venttype to category
# big_df.loc[big_df.loc[:,'label']=='vent_recieved','value']= big_df.loc[big_df.loc[:,'label']=='vent_recieved','value'].astype('category')
# #big_df.loc[big_df.loc[:,'label']=='vent_recieved','value'].apply(astype('category'))
# big_df.loc[big_df.loc[:,'label']=='vent_recieved','value'] = pd.Categorical(big_df.loc[big_df.loc[:,'label']=='vent_recieved','value'])
def category_mapper(big_df):
    #converting categories to integers
    mapper={'Mech':2 , 'Oxygen': 1, 'None': 0}
    big_df.loc[big_df.loc[:,'label']=='vent_recieved','value']=big_df.loc[big_df.loc[:,'label']=='vent_recieved','value'].replace(mapper).copy()

    #gender_index=pd.factorize(big_df.loc[big_df.loc[:,'label']=='gender','value'])[1]
    mapper={'F':0 , 'M': 1}
    big_df.loc[big_df.loc[:,'label']=='gender','value']=big_df.loc[big_df.loc[:,'label']=='gender','value'].replace(mapper).copy()

    #leukocyte_index=pd.factorize(big_df.loc[big_df.loc[:,'label']=='leukocyte','value'])[1]
    mapper={'Neg/Not_tested':0 , 'pos': 1}
    big_df.loc[big_df.loc[:,'label']=='leukocyte','value']=big_df.loc[big_df.loc[:,'label']=='leukocyte','value'].replace(mapper).copy()
    #     pd.factorize(big_df.loc[big_df.loc[:,'label']=='leukocyte','value'])[0] 

    #nitrite_index=pd.factorize(big_df.loc[big_df.loc[:,'label']=='nitrite','value'])[1]
    mapper={'Neg/Not_tested':0 , 'pos': 1}
    big_df.loc[big_df.loc[:,'label']=='nitrite','value']=big_df.loc[big_df.loc[:,'label']=='nitrite','value'].replace(mapper).copy()
    #     pd.factorize(big_df.loc[big_df.loc[:,'label']=='nitrite','value'])[0] 
    
    
    # changing pao2/fio2 ratio to a category
    """
    A PaO2/FiO2 ratio less than or equal to 200 is necessary for the diagnosis of acute respiratory distress syndrome by the AECC criteria.[6] 
    The more recent Berlin criteria defines mild ARDS at a ratio of <300.

    A PaO2/FiO2 ratio less than or equal to 250 is one of the minor criteria for severe community acquired pneumonia (i.e., possible indication for inpatient treatment).

    A PaO2/FiO2 ratio less than or equal to 333 is one of the variables in the SMART-COP risk score for intensive respiratory or vasopressor support in community-acquired pneumonia.
    """

    
    pd.to_numeric(big_df[big_df['label']=="pao2fio2ratio"]['value'])
    labels= ['0-200', '201-333', "334-475","476+"]
    bins = pd.IntervalIndex.from_tuples([(0, 200), (200, 333), (333, 475),(475,3000)])

    big_df.loc[big_df['label']=="pao2fio2ratio",'value']=pd.cut(pd.to_numeric(big_df.loc[big_df['label']=="pao2fio2ratio",'value']), bins,right=False, labels=labels)
    big_df.loc[big_df['label']=="pao2fio2ratio",'value'].value_counts()

    
    return(big_df)

big_df_train=category_mapper(big_df_train)
big_df_test=category_mapper(big_df_test)<jupyter_output>time: 3.73 s<jupyter_text>calc median/iqr for standardizationtake all non-categorical variables for HEALTHY PATIENTS and calculate the median and IQR for them. then will use this to make z scores via: x-median/IQR  june 13: changed sparse to catgorical<jupyter_code>continuous=['daily_sofa',
            'lactate',
            'mingcs',
            'diasbp',
            'heartrate',
            'meanartpress',
            'resprate',
            'sysbp',
            'temperature',
            'hemoglobin',
            'platelet',
            'wbc',
            'calcium',
            'glucose',
            'ph',
            'bicarbonate',
            'bun',
            'chloride',
            'creatinine',
            'inr',
            'potassium',
            'ptt',
            'sodium',
            'bilirubin',
            'spo2',
            'sum_elix']


onetime=['first_admit_age','height','weight']

vaso_active=['phenylephrine',
            'norepinephrine',
            'vasopressin',
            'dobutamine',
            'dopamine',
            'epinephrine'] 

ordinal=[
            'leukocyte',
            'nitrite',
            'vent_recieved',
            'o2_flow',
            'rrt',
            'pao2fio2ratio',
            'cancer_elix',
            "any_vasoactives",
            'bands', #added 6/13/19
            'pco2' #added 6/13/19
]

categorical=[
            "ethnicity",
            'gender'
]

def median_label_fxn(big_df):
    global final_pt_df2
    
    healthy_pt=list(final_pt_df2[final_pt_df2['final_bin']=="C_neg/A_partial"]['icustay_id'])

    #filter to only healthy patients filter
    ##splitting big_df, making a copy and restricting it to all values that will be usd in standardizing
    healthy_df=big_df[big_df['label'].isin(continuous+onetime)].copy()
    healthy_df['value']= pd.to_numeric(healthy_df['value']) #converting to numeric 
    healthy_df=healthy_df[healthy_df['icustay_id'].isin(healthy_pt)].copy() #only numerical values for cneg/ab partial pt
    
    #finding the last 24 hours of each healthy patient.
    healthy_pt_end=pd.DataFrame(healthy_df.groupby("icustay_id")['delta'].max())
    healthy_pt_end["start"]=healthy_pt_end['delta']- pd.to_timedelta("1 day 00:00:00")
    healthy_pt_end=healthy_pt_end.rename(columns={"delta":"end"}).reset_index()
    healthy_df= pd.merge(healthy_df,healthy_pt_end, left_on="icustay_id", right_on="icustay_id", how="left" ) #now have the last 24 hours annotated for each patient as start and end.
    
    #calculating medians and iqr for each label based on healthy patient's last 24 hours in icu
    median_label=pd.DataFrame((healthy_df.groupby("label")['value'].median())).reset_index()
    median_label=median_label.rename(columns={'value':"median"})
    iqr_label=pd.DataFrame((healthy_df.groupby("label")['value'].quantile(0.75)-healthy_df.groupby("label")['value'].quantile(0.25))).reset_index()
    iqr_label=iqr_label.rename(columns={'value':"iqr"})
    median_label=pd.merge(median_label,iqr_label)  #final median df
    return(median_label)

def combine_vasoactives(big_noCat, median_label):
    global vaso_active
    ##combining vasoactives
    vaso_active_df=big_noCat[big_noCat['label'].isin(vaso_active)].groupby('icustay_id')['value'].max().reset_index()
    vaso_active_df['uom']="y/n"
    vaso_active_df['label']="any_vasoactives"
    vaso_active_df['delta']=pd.to_timedelta("0days")
    vaso_active_df['source']="any_vasoactives"

    vaso_active_df=pd.merge(vaso_active_df, final_pt_df2[["icustay_id",'subject_id',"t_0"]], how="left", left_on="icustay_id", right_on="icustay_id")
    vaso_active_df=pd.merge(vaso_active_df, median_label, how="left").fillna(0)
    vaso_active_df['standardize']=vaso_active_df['value']
    vaso_active_df['raw_value']=vaso_active_df['value']
    vaso_active_df.head()

    # # #grabing the rest of the variables not suitable for range or mean/std
    big_noCat=pd.concat([big_noCat, vaso_active_df], sort=False)
    return(big_noCat)

def standardization_fxn(big_df):
    global continuous, onetime, vaso_active, ordinal, categorical
    
    median_label=median_label_fxn(big_df)
    median_label
    median_label_nocat=median_label.loc[median_label.loc[:,'label'].isin(continuous),:].copy()
    median_label_cat=median_label.loc[median_label.loc[:,'label'].isin(continuous),:].copy()
    
    ### dataformatting: convert all dtypes to a numeric type that pereserves nan. 
    #splitting categorical, ordinal and continuous
    big_categorical= big_df.loc[big_df.loc[:,'label'].isin(categorical),:].copy() 

    #continuous and ordinal variables
    big_noCat= big_df.loc[big_df.loc[:,'label'].isin(continuous),:].copy() 
    big_noCat['value']= big_noCat['value'].apply(pd.to_numeric, args=('coerce',)) #instead of convert to float, may preserve nan's better. 
    
    ### adding a standardized value (x-median)/iqr  where median is of the last 24 hours in time window for culture neg/ ab partial patients 
    big_noCat=pd.merge(big_noCat, median_label, how="left") 
    #loging values
    big_noCat['median']= np.log(big_noCat['median']+1.0)
    big_noCat['iqr']= np.log(big_noCat['iqr']+1.0)
    big_noCat['raw_value']=big_noCat['value']
    big_noCat['value']=np.log(big_noCat['value']+1.0)
    
    big_noCat['standardize']=((big_noCat['value']-big_noCat['median'])/big_noCat['iqr']).fillna(0) #standardize is log standardized

    #making an ordinal df to concat on
    ord_df=big_df.loc[big_df.loc[:,'label'].isin(ordinal+vaso_active),:].copy()
    ord_df['raw_value']=ord_df['value']
    ord_df['standardize']=ord_df['value']
    ord_df['median']=None
    ord_df['iqr']=None

    big_noCat=pd.concat([big_noCat,ord_df], sort=False)
    
    #making all ordinal values in standardize equal to unstandardized
    big_noCat.loc[big_noCat['label'].isin(ordinal+vaso_active),'standardize']=big_noCat.loc[big_noCat['label'].isin(ordinal+vaso_active),'value']  #do i need to add sparse?
    #ig_noCat.head()
    
    big_noCat['standardize']= big_noCat['standardize'].apply(pd.to_numeric, args=('coerce',)) #errors occuring downstream due to not having numeric, trying this 4/18/19
    
    ##last step, combining vasoactives into 1 feature
    big_noCat=combine_vasoactives(big_noCat, median_label)  

    return(big_noCat, big_categorical)
median_label_fxn(big_df_train)

##standardizing and train/test splitting data

noCat_df_train, cat_df_train = standardization_fxn(big_df_train)
noCat_df_train['delta']= pd.to_timedelta(noCat_df_train['delta'])

noCat_df_test, cat_df_test = standardization_fxn(big_df_test)
noCat_df_test['delta']= pd.to_timedelta(noCat_df_test['delta'])

#fixing one patient with delta=4 sofa
noCat_df_train.loc[noCat_df_train['delta']==pd.Timedelta("4 days"), 'delta']=pd.Timedelta("3 days")
#fixing one patient with delta=4 sofa
noCat_df_test.loc[noCat_df_test['delta']==pd.Timedelta("4 days"), 'delta']=pd.Timedelta("3 days")
noCat_df_train.head()
noCat_df_train[noCat_df_train['label']=='sum_elix']
len(noCat_df_train['label'].unique()) #42, should be 28
noCat_df_train['icustay_id'].unique()[1]
#fixing one patient with delta=4 sofa
noCat_df_train.loc[noCat_df_train['delta']==pd.Timedelta("4 days"), 'delta']=pd.Timedelta("3 days")
#fixing one patient with delta=4 sofa
noCat_df_test.loc[noCat_df_test['delta']==pd.Timedelta("4 days"), 'delta']=pd.Timedelta("3 days")<jupyter_output>time: 857 ms<jupyter_text>messing with timeseries vis* resampling different time windows* forward fill * cat vs nocat* visualization<jupyter_code>noCat_df_train['delta'].describe()
#trying to find a patient without a 0:00:00 and 3day time
noCat_df_train.loc[(noCat_df_train.loc[:,'icustay_id']==noCat_df_train['icustay_id'].unique()[5]),['delta']].describe()
(noCat_df_train.groupby('icustay_id')['delta'].min()).min()
#Good news, everyone has atleast a 0:00:00 timepoint
(noCat_df_train.groupby('icustay_id')['delta'].max()).min()
#but not everyone has a 3day timepoint.<jupyter_output><empty_output><jupyter_text>trying to pivot table before any resample<jupyter_code>#one patient and all variable
test_df1=noCat_df_train.loc[(noCat_df_train.loc[:,'icustay_id']==200030.0),['delta','value','label']].copy()

#one patient and one variable
test_df2=noCat_df_train.loc[(noCat_df_train.loc[:,'label']=='diasbp')&(noCat_df_train.loc[:,'icustay_id']==200030.0),['delta','value']].copy()

hourly=test_df2.set_index('delta').resample("H").max().fillna(method="ffill") #max and 
#hourly=test_df2.resample("H",on='delta').count() #same as above but don't need to set delta to index.
test_df1.head()
test_df2.head()
hourly.head()
hourly['value'].plot(style=[':','--','-']) #work<jupyter_output><empty_output><jupyter_text>step2: write fxn to standardize timeframe to 0:00:00 to 2days 23:00:00 for hourly after resampling. (this code isn't used in final).<jupyter_code>def timeframe_fix_fxn(df):
    """
    this fxn appends a starting and ending time onto dataframe.
    df is a dataframe with delta and value for a specific label.
    """
    ##don't need this because eveyrone has atleast 1 0:00:00 timepoint.
#     #adding in start time of 0:00:00: 
#     ##making a df with 0:00 timedelta with none value and appending this to first row
#     if (df.sort_values("delta")['delta'].iloc[0] != pd.Timedelta("00:00:00")):
#         df2=pd.DataFrame([[pd.Timedelta('00:00:00'),None]], columns=['delta','value'])
#         df=df.append(df2) 
#         ##alternative way of doing it:
# #         df.loc[-1] = [pd.Timedelta("00:00:00"), None]  # adding a row
# #         df.index = df.index + 1  # shifting index
# #         df = df.sort_index()  # sorting by index
#     else:
#         pass
    
    #adding in end time of '2 days 23:00:00'
    if (df.sort_values("delta")['delta'].iloc[-1] != pd.Timedelta('3 days 00:00:00')):
        df3=pd.DataFrame([[pd.Timedelta('3 days 00:00:00'),None]], columns=['delta','value'])
        df=df.append(df3)
    else:
        pass
    
    df= df.sort_values('delta')
    return(df)

#adding start and endtime if not already there. works for single variable and single patient.
test_df3=timeframe_fix_fxn(test_df2) #adding on 0:00:00 start and 2day 23:00:00 end. 
test_df3.head()<jupyter_output><empty_output><jupyter_text>problem: only works for single variable, doesn't work with many variables. solution: i'll just add a delta value to the initial dataset. (currently only works for this single patient, need to expand if i go to all patients)note, by adding a Null bicarbonate value at t_0+72 it records a value there and is thus is used as the endpoint to the resampling fxn. if there is infact a value there, then it will be taken instead of the null (due to the max fxn).<jupyter_code># #adding a row with null values and final timepoint
# df= pd.DataFrame(test_df1.iloc[-1].T)
# df.iloc[:,0]=None
# df=pd.DataFrame(df).T
# df['delta']=pd.Timedelta("2 days 23:00:00")
# df['label']="bicarbonate"

# test_df=test_df1.append(df)
# test_df1['delta'].describe()
noCat_df_train[noCat_df_train['label']=='bicarbonate'].head(2)
#first getting a row with bicarbonate from noCat_df
bicarb=noCat_df_train.loc[noCat_df_train['label']=='bicarbonate',:].head(1)

#making a df for each icustay and adding appropriate values for a 3day final timepoint with None values.
df= pd.DataFrame(noCat_df_train.drop_duplicates('icustay_id'))#.iloc[-1].T)

df['uom']=bicarb['uom'].values[0]
df['label']=bicarb['label'].values[0]
df['value']=None
df['delta']=pd.Timedelta("3 days")
df['source']=bicarb['source'].values[0]
df['median']=bicarb['median'].values[0]
df['iqr']=bicarb['iqr'].values[0]
df['raw_value']=None
df['standardize']=None
#df

# appending this onto nocat dataset with ONLY continuous (removed ordinal)
noCat_df_train2= noCat_df_train[noCat_df_train['label'].isin(continuous)].copy()
noCat_df_train2=noCat_df_train2.append(df)

#testing the minimum  of all icustays max deltas. 
(noCat_df_train2.groupby('icustay_id')['delta'].max()).min()
noCat_df_train2['icustay_id'].nunique() #2418291 rows.
#5316 icustay_ids
noCat_df_train2.head()
noCat_df_train2['label'].unique()<jupyter_output><empty_output><jupyter_text>step3: visualize the resampled variables for a single patient<jupyter_code>maxCol=lambda x: max(x.min(), x.max(), key=abs)
#df.apply(maxCol,axis=1)
grouper = noCat_df_train2[['delta','standardize','label','icustay_id']].groupby(['icustay_id',pd.Grouper(key="delta",freq='4H'), 'label'])

grouper['standardize'].apply(maxCol).unstack(["label"]).head(5)
#grouper['standardize'].max().unstack(["label"]).isna().sum()
#test_unstacked=grouper['standardize'].max().unstack("label")
test_unstacked=grouper['standardize'].apply(maxCol).unstack("label")
test_unstacked.head(19) #19 delta values
##takes 10 minutes to run, resamples the data in 4 hour blocks and puts na for values unfilled. every icustay_id now has a row for every 4 hour block. 
restacked=test_unstacked.groupby('icustay_id').resample('4H',level=-1).apply(maxCol) #level =-1 : corresponds to the delta multiindex. 
restacked_ffilled=restacked.groupby('icustay_id').fillna(method='ffill') #fills in na using feed forward. 
##issue is, if there is no value previously seen, then na still exists. for these na before a value is seen, a 0 is filled in, to represent healthy median value. 
restacked_ffilled=restacked_ffilled.fillna(0)# 
-restacked_ffilled.head()
###code graveyard

# #test_unstacked.reset_index().groupby('icustay_id').resample("4H").max().fillna(method='ffill')
# #test_unstacked.groupby(pd.Grouper(freq='4H', level=1)).max()
# #test_unstacked.loc[299914.0,:]#.groupby(['icustay_id',pd.Grouper(pd.Grouper(freq='4H', level=1))])
# #test_unstacked.groupby(['icustay_id',pd.Grouper(freq='4H', level=-1)]).resample("4H").max().fillna(method='ffill')#.max()

# # def using_reset_index(df):
# #     df = df.reset_index(level=[0, 1])
# #     return df.groupby(['icustay_id','delta']).resample('4H').max().fillna(method='ffill').max()

# #test_unstacked.reset_index(level=[0, 1]).groupby('icustay_id').resample

# #test_unstacked.groupby('icustay_id').resample('4H',level=-1).max().fillna(method='ffill').max()

# #test_unstacked.groupby('icustay_id').resample('4H',level=-1).apply(maxCol).fillna(method='ffill').apply(maxCol)

# ##takes 10 minutes to run, resamples the data in 4 hour blocks and 
# restacked=test_unstacked.groupby('icustay_id').resample('4H',level=-1).apply(maxCol) #level =-1 : corresponds to the delta multiindex. 


# #.apply(maxCol)

# #using_reset_index(test_unstacked)
# #test_unstacked.resample("4H").max().fillna(method='ffill')#.max()<jupyter_output>time: 11min 34s<jupyter_text>now need to turn into a tensor and get a y label for each icustay_id.* [samples, timesteps, features]: 5316, 26, 19<jupyter_code>m,n = len(restacked_ffilled.index.levels[0]), len(restacked_ffilled.index.levels[1])
xtrain = restacked_ffilled.values.reshape(m,n,-1).swapaxes(1,2)
xtrain.shape
np.save("rnn/lstm_xtrain", xtrain, allow_pickle=True, fix_imports=True)
np.save("rnn/lstm_ytrain", ytrain, allow_pickle=True, fix_imports=True)<jupyter_output>time: 247 ms<jupyter_text>wrapping it all in functions<jupyter_code>def prep_for_resample(noCat_df):
    #first getting a row with bicarbonate from noCat_df
    bicarb=noCat_df.loc[noCat_df['label']=='bicarbonate',:].head(1)

    #making a df for each icustay and adding appropriate values for a 3day final timepoint with None values.
    df= pd.DataFrame(noCat_df.drop_duplicates('icustay_id'))#.iloc[-1].T)

    df['uom']=bicarb['uom'].values[0]
    df['label']=bicarb['label'].values[0]
    df['value']=None
    df['delta']=pd.Timedelta("3 days")
    df['source']=bicarb['source'].values[0]
    df['median']=bicarb['median'].values[0]
    df['iqr']=bicarb['iqr'].values[0]
    df['raw_value']=None
    df['standardize']=None

    # appending this onto nocat dataset with ONLY continuous (removed ordinal)
    noCat_df= noCat_df[noCat_df['label'].isin(continuous)].copy()
    noCat_df=noCat_df.append(df)
    return(noCat_df)

def resampler(df2):
    maxCol=lambda x: max(x.min(), x.max(), key=abs)
    
    grouper = df2[['delta','standardize','label','icustay_id']].groupby(['icustay_id',pd.Grouper(key="delta",freq='4H'), 'label'])
    test_unstacked=grouper['standardize'].apply(maxCol).unstack("label")

    ##takes 10 minutes to run, resamples the data in 4 hour blocks and puts na for values unfilled. every icustay_id now has a row for every 4 hour block. 
    restacked=test_unstacked.groupby('icustay_id').resample('4H',level=-1).apply(maxCol) #level =-1 : corresponds to the delta multiindex. 

    restacked_ffilled=restacked.groupby('icustay_id').fillna(method='ffill') #fills in na using feed forward. 
    ##issue is, if there is no value previously seen, then na still exists. for these na before a value is seen, a 0 is filled in, to represent healthy median value. 
    restacked_ffilled=restacked_ffilled.fillna(0)# 
    
    return(restacked_ffilled)

def main(test):
    if test==True:
        df2=prep_for_resample(noCat_df_test)    
        restacked_df=resampler(df2)
        
        m,n = len(restacked_df.index.levels[0]), len(restacked_df.index.levels[1]) #(n_patients, n_timepoints)
        x_test = restacked_df.values.reshape(m,n,-1)#restacked_ffilled_df2.values.reshape(m,-1,n)#.swapaxes(1,2)
        
        #
        y_test= pd.merge(pd.DataFrame({'icustay_id':restacked_df.index.levels[0]}), final_pt_df2[['icustay_id','final_bin']])
        y_test.loc[y_test['final_bin']=='C_pos/A_full','final_bin']=1
        y_test.loc[y_test['final_bin']=='C_neg/A_partial','final_bin']=0
        y_test=y_test['final_bin'].values
        
        return(restacked_df, x_test, y_test)

    else:
        df2=prep_for_resample(noCat_df_train)    
        restacked_df=resampler(df2)
        
        
        train_patients=final_pt_df2.loc[final_pt_df2.loc[:,"subject_id"].isin(train_subject),['icustay_id','subject_id','final_bin']]
        train_patients['final_bin']=pd.factorize(train_patients['final_bin'])[0]
        train_patients_maxsub=train_patients.loc[train_patients.groupby('subject_id')['final_bin'].idxmax(),:]

        # 70/30 train/test set split with 12345=seed, splitting on max final bin of each SUBJECT_ID
        train_val, val = train_test_split(train_patients_maxsub, test_size=0.2, random_state=12345, 
                                       stratify=train_patients_maxsub['final_bin'])
        # generate list of each SUBJECT_ID in each split
        train_val_subject=list(train_val['subject_id'])
        val_subject=list(val['subject_id'])

        train_val_icu=list(two_class_icu.loc[two_class_icu['subject_id'].isin(train_val_subject),'icustay_id'] )
        val_icu=list(two_class_icu.loc[two_class_icu['subject_id'].isin(val_subject),'icustay_id'] )
        # # #filtering big_df on train subjects and test subjects to get my train/test splits.
        restacked_train= restacked_df[restacked_df.index.get_level_values('icustay_id').isin(train_val_icu)]
        restacked_val= restacked_df[restacked_df.index.get_level_values('icustay_id').isin(val_icu)]

        ###filtering x
        m,n = restacked_train.reset_index()['icustay_id'].nunique(), restacked_train.reset_index()['delta'].nunique() #(n_patients, n_timepoints)
        x_train = restacked_train.values.reshape(m,n,-1)#restacked_ffilled_df2.values.reshape(m,-1,n)#.swapaxes(1,2)

        m,n = restacked_val.reset_index()['icustay_id'].nunique(), restacked_val.reset_index()['delta'].nunique()
        #len(restacked_val.index.levels[0]), len(restacked_val.index.levels[1]) #(n_patients, n_timepoints)
        x_val = restacked_val.values.reshape(m,n,-1)#restacked_ffilled_df2.values.reshape(m,-1,n)#.swapaxes(1,2)
             
        y_train= pd.merge(pd.DataFrame({'icustay_id':list(restacked_train.reset_index()['icustay_id'].unique())}), final_pt_df2[['icustay_id','final_bin']])
        y_train.loc[y_train['final_bin']=='C_pos/A_full','final_bin']=1
        y_train.loc[y_train['final_bin']=='C_neg/A_partial','final_bin']=0
        y_train=y_train['final_bin'].values

        y_val= pd.merge(pd.DataFrame({'icustay_id':list(restacked_val.reset_index()['icustay_id'].unique())}), final_pt_df2[['icustay_id','final_bin']])
        y_val.loc[y_val['final_bin']=='C_pos/A_full','final_bin']=1
        y_val.loc[y_val['final_bin']=='C_neg/A_partial','final_bin']=0
        y_val=y_val['final_bin'].values

        return(restacked_df, x_train, x_val, y_train, y_val )
restacked_df, xtrain_1, ytrain_1= main(test=False)<jupyter_output>time: 19min 31s<jupyter_text>qc and inline code running<jupyter_code># train_patients=final_pt_df2.loc[final_pt_df2.loc[:,"subject_id"].isin(train_subject),['icustay_id','subject_id','final_bin']]
# train_patients['final_bin']=pd.factorize(train_patients['final_bin'])[0]
# train_patients_maxsub=train_patients.loc[train_patients.groupby('subject_id')['final_bin'].idxmax(),:]

# # 70/30 train/test set split with 12345=seed, splitting on max final bin of each SUBJECT_ID
# train_val, val = train_test_split(train_patients_maxsub, test_size=0.2, random_state=12345, 
#                                stratify=train_patients_maxsub['final_bin'])
# # generate list of each SUBJECT_ID in each split
# train_val_subject=list(train_val['subject_id'])
# val_subject=list(val['subject_id'])

# train_val_icu=list(two_class_icu.loc[two_class_icu['subject_id'].isin(train_val_subject),'icustay_id'] )
# val_icu=list(two_class_icu.loc[two_class_icu['subject_id'].isin(val_subject),'icustay_id'] )
# # # #filtering big_df on train subjects and test subjects to get my train/test splits.
# restacked_train= restacked_df[restacked_df.index.get_level_values('icustay_id').isin(train_val_icu)]
# restacked_val= restacked_df[restacked_df.index.get_level_values('icustay_id').isin(val_icu)]
# # restacked_train= restacked_df.loc[train_val_icu,:].copy()#restacked_df.loc[restacked_df.loc[:,'icustay_id'].isin(train_val_icu),:].copy()
# # restacked_val= restacked_df.loc[val_icu,:].copy()#restacked_df.loc[restacked_df.loc[:,'icustay_id'].isin(val_icu),:].copy()

print(two_class_icu['icustay_id'].nunique(), two_class_icu['subject_id'].nunique()) #7588 icustay
print(5316+2272, 4776+2048)
# two_class_icu.loc[two_class_icu['subject_id'].isin(train_subject),'subject_id'].nunique()  #4776 subject_id
# two_class_icu.loc[two_class_icu['subject_id'].isin(test_subject),'subject_id'].nunique() #2048 subject_id
# two_class_icu.loc[two_class_icu['subject_id'].isin(train_subject),'icustay_id'].nunique() #5316 icustay_ids
# two_class_icu.loc[two_class_icu['subject_id'].isin(test_subject),'icustay_id'].nunique() #2272 icustay_ids

len(train_val_icu) + len(val_icu) #4776= 956 + 3820
#5316= 4263+1053
# print(len(list(output_df.index.levels[0])), len(set(output_df.index.levels[0])))
# restacked_train.reset_index()['delta'].nunique()
m,n = restacked_train.reset_index()['icustay_id'].nunique(), restacked_train.reset_index()['delta'].nunique() #(n_patients, n_timepoints)
xtrain_2 = restacked_train.values.reshape(m,n,-1)#restacked_ffilled_df2.values.reshape(m,-1,n)#.swapaxes(1,2)

m,n = restacked_val.reset_index()['icustay_id'].nunique(), restacked_val.reset_index()['delta'].nunique()
#len(restacked_val.index.levels[0]), len(restacked_val.index.levels[1]) #(n_patients, n_timepoints)
xval_2 = restacked_val.values.reshape(m,n,-1)#restacked_ffilled_df2.values.reshape(m,-1,n)#.swapaxes(1,2)

y_train= pd.merge(pd.DataFrame({'icustay_id':list(restacked_train.reset_index()['icustay_id'].unique())}), final_pt_df2[['icustay_id','final_bin']])
y_train.loc[y_train['final_bin']=='C_pos/A_full','final_bin']=1
y_train.loc[y_train['final_bin']=='C_neg/A_partial','final_bin']=0
y_train=y_train['final_bin'].values

y_val= pd.merge(pd.DataFrame({'icustay_id':list(restacked_val.reset_index()['icustay_id'].unique())}), final_pt_df2[['icustay_id','final_bin']])
y_val.loc[y_val['final_bin']=='C_pos/A_full','final_bin']=1
y_val.loc[y_val['final_bin']=='C_neg/A_partial','final_bin']=0
y_val=y_val['final_bin'].values
np.save("rnn/lstm_xtrain", xtrain_2, allow_pickle=True, fix_imports=True)
np.save("rnn/lstm_xval", xval_2, allow_pickle=True, fix_imports=True)
np.save("rnn/lstm_ytrain", y_train, allow_pickle=True, fix_imports=True)
np.save("rnn/lstm_yval", y_val, allow_pickle=True, fix_imports=True)
y_val.shape
np.save("rnn/lstm_xtest", xtest3, allow_pickle=True, fix_imports=True)
np.save("rnn/lstm_ytest", ytest, allow_pickle=True, fix_imports=True)<jupyter_output>time: 115 ms<jupyter_text>visualizing data<jupyter_code>#
y= pd.merge(pd.DataFrame({'icustay_id':restacked_ffilled.index.levels[0]}), final_pt_df2[['icustay_id','final_bin']])
y.loc[y['final_bin']=='C_pos/A_full','final_bin']=1
y.loc[y['final_bin']=='C_neg/A_partial','final_bin']=0
ytrain=y['final_bin'].values
ytrain
xtrain[0]
xtrain[0,0,:]
xtrain[0, :,0] #timestep 0 for first patient: 
xtrain[0, :,0] #timestep 0 for first patient: 
# print(i)
# print(Y_test[i])
plt.imshow(xtest3[1])
#plt.imshow(arr[1])
plt.show()
plt.imshow(arr[2])
plt.show()
arr[1].shape
# #test_unstacked.fillna(value=0).count()#.count()#.sum()
# test_unstacked.shape[0]*test_unstacked.shape[1]
#noCat_df_train2['delta'].nunique()
#test_unstacked.shape[0]/noCat_df_train2['icustay_id'].nunique()
#test_unstacked.isna().sum().sum()
#1231122/2013960<jupyter_output><empty_output><jupyter_text>need to find the best resampling timepoint:* get the % null values over a few resampling periods.<jupyter_code>len(grouper['value'].max())
grouper['value'].max()#.isna().sum()
# def percent_na(grouper):
#     grouped_max=grouper['value'].max()
#     sum_na=grouped_max.isna().sum()
#     total_val=grouped_max.count()
#     print("sumna:{}, total_values:{}, percent:{}".format(sum_na, total_val, 100*(sum_na/total_val)))

# percent_na(grouper)<jupyter_output>sumna:5312, total_values:874271, percent:0.6075919251582175
time: 1min 49s
{'fraction_non_alphanumeric': 0.16321999072048785, 'fraction_numerical': 0.023165639292105786, 'mean_word_length': 5.77785265049416, 'pattern_counts': {'":': 1, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '426855', 'license': 'MIT', 'n_tokens_mistral': 11893, 'n_tokens_neox': 11228, 'n_words': 1740, 'path': 'notebooks/data_exploration_notebooks/07.051-timeseries_vis.ipynb', 'repo_name': 'geickelb/mimiciii-antibiotics-modeling', 'size': '30174'}