<jupyter_start><jupyter_text>Aligning time<jupyter_code>def align_time(df):
    '''Align first day of notes per patient as being day 0, and calculates all consecutive days from that.'''
    
    df['year'] = df['Notitiedatum'].str.split('-').str[0]
    df['month'] = df['Notitiedatum'].str.split('-').str[1]
    df['day'] = df['Notitiedatum'].str.split('-').str[2]

    #array with idx for first occurrence of a patient (trick from https://stackoverflow.com/questions/41255215/pandas-find-first-occurrence)
    first_occ= [df.MDN.eq(patient).idxmax() for patient in df.MDN.unique()]

    deltas = []
    counter = 0
    for idx in first_occ: 
        f_date = date(int(df['year'][idx]), int(df['month'][idx]), int(df['day'][idx])) #reference date (first visit)
        df_patient = df[df.MDN ==df.MDN[idx]]
        counter +=  len(df_patient)
        for year,month,day in zip(df_patient['year'],df_patient['month'],df_patient['day']): #smaller dataframe with only a given patient data
            l_date = date(int(year), int(month), int(day)) #put i to generalize
            deltas.append(str(l_date-f_date).split(" ")[0].split(":")[0])
            if int(str(l_date-f_date).split(" ")[0].split(":")[0]) > 460: 
                print(str(l_date-f_date).split(" ")[0].split(":")[0])
                print(year, month, day)
    df['Days_count'] = deltas
    print(len(deltas),len(df['Notitiedatum']),counter)

    #encode days_count as int64
    labelenc = LabelEncoder()
    df.iloc[:,-1] = labelenc.fit_transform(df.iloc[:,-1].values)
    
    return df 

df_fildate = align_time(df_fildate)      
#updating dataframe with all filters included
m = df_fildate.MDN.isin(cross_id_dom50.MDN)
df_final = df_fildate[m]

labelenc = LabelEncoder()
df_final.iloc[:,-1] = labelenc.fit_transform(df_final.iloc[:,-1].values)<jupyter_output><empty_output><jupyter_text>- Use only the first admission data (so not use the outliers that happens months later when the person goes back for a checkup)<jupyter_code>#difference between day of note and previous date of note. for first note of patient this value is defined as zero.
df_final['Dif'] = df_final.Days_count.diff(periods=1)
df_final['Dif'] = df_final['Dif'].clip(lower=0, upper=None, axis=0)
df_final.iloc[:,-1] = labelenc.fit_transform(df_final.iloc[:,-1].values)
#check how many days without any note indicates the person is not anymore admitted. Initial filter is if it is 3 days in a row without notes, notes after are disregarded
#from this point on the trimmed dataframe based on final dataset will be called df for simplicity
df = df_final[df_final['Dif'] < 3]
print(df.shape, df_final.shape)<jupyter_output>(17912, 20) (20662, 20)<jupyter_text>- Interpolating data per domain (in principle for ADM only) and creating time series features<jupyter_code>#checking missing values. Here we focus on ADM_lvl mainly
df.isnull().sum()
# Interpolate the dataset based on previous/next values..
def impute_interpolate(dataset, col):
    dataset[col] = dataset[col].interpolate()
    # And fill the initial data points if needed:
    dataset[col] = dataset[col].fillna(method='bfill')
    return dataset
def add_lag(df):
    '''Interpolates and creates new features by shifting window 1,2 and 3 instances within each patient.'''
    
    first_occ = [df.MDN.eq(patient).idxmax() for patient in df.MDN.unique()]
    #creating different lags that can affect the time series
    df['lag1_ADM'] = df.loc[:, 'lag1_ADM'] = 0
    df['lag2_ADM'] = df.loc[:, 'lag2_ADM'] = 0
    df['lag3_ADM'] = df.loc[:, 'lag3_ADM'] = 0

    counter = 0

    for idx in first_occ: 
        df_patient = df[df.MDN == df.MDN[idx]]
        counter +=  len(df_patient)
        for domain in df_patient['ADM_lvl']: #smaller dataframe with only a given patient data
            # Interpolate the column based on previous/next values for ADM
            df['ADM_lvl'].loc[df.MDN == df.MDN[idx]] = df_patient['ADM_lvl'].interpolate()
            df['ADM_lvl'].loc[df.MDN == df.MDN[idx]] = df_patient['ADM_lvl'].fillna(method='bfill')

            #by shifting one element.  
            df['lag1_ADM'].loc[df.MDN == df.MDN[idx]] = df_patient.ADM_lvl.shift(1)
            df['lag2_ADM'].loc[df.MDN == df.MDN[idx]] = df_patient.ADM_lvl.shift(2)
            df['lag3_ADM'].loc[df.MDN == df.MDN[idx]] = df_patient.ADM_lvl.shift(3)

    df.isnull().sum()

    labelenc = LabelEncoder()
    #encode days_count as int64
    df.iloc[:,-1] = labelenc.fit_transform(df.iloc[:,-1].values)
    df.iloc[:,-2] = labelenc.fit_transform(df.iloc[:,-2].values)
    df.iloc[:,-3] = labelenc.fit_transform(df.iloc[:,-3].values)
    return df

df = add_lag(df)<jupyter_output><empty_output><jupyter_text>- Creating feature with average result of all domains per instance<jupyter_code>#creating feature that is average of all domains
columns = ['ADM_lvl', 'ATT_lvl', 'BER_lvl', 'ENR_lvl', 'ETN_lvl', 'FAC_lvl', 'INS_lvl', 'MBW_lvl', 'STM_lvl']  
df['avg_domains'] = df[columns].mean(axis=1)
df<jupyter_output><empty_output><jupyter_text>- Create features using different window techniques<jupyter_code>def rolling_and_expanding_mean(df):
    '''Rolling window feature: the size of the window is constant while the window slides forward in time. 
    calculating some statistical values based on past values using a rolling window method.
     
    Expanding window: an advanced version of the rolling window technique. 
    This feature takes all the past values into account.'''

    first_occ = [df.MDN.eq(patient).idxmax() for patient in df.MDN.unique()]

    counter = 0

    for idx in first_occ: 
        df_patient = df[df.MDN == df.MDN[idx]]
        counter +=  len(df_patient)
        for domain in df_patient['ADM_lvl']: #smaller dataframe with only a given patient data
            
            #mean, sum, min, max value

            df['rolling_mean3'] = df['ADM_lvl'].rolling(window=3).mean()
            df['rolling_mean7'] = df['ADM_lvl'].rolling(window=7).mean()

            df['rolling_sum3'] = df['ADM_lvl'].rolling(window=3).sum()
            df['rolling_sum7'] = df['ADM_lvl'].rolling(window=7).sum()

            df['rolling_min3'] = df['ADM_lvl'].rolling(window=3).min()
            df['rolling_min7'] = df['ADM_lvl'].rolling(window=7).min()

            df['rolling_max3'] = df['ADM_lvl'].rolling(window=3).max()
            df['rolling_max7'] = df['ADM_lvl'].rolling(window=7).max()

            
            df['expanding_mean'] = df['ADM_lvl'].expanding(2).mean()

    df.isnull().sum()
    return df

df = rolling_and_expanding_mean(df)
df.to_csv('/Users/brunaguedes/Documents/GitHub/a-proof-time-series/processed/feature_engineering.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1474502766561986, 'fraction_numerical': 0.014505757439808583, 'mean_word_length': 3.7805575411007863, 'pattern_counts': {'":': 3, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '451608', 'license': 'Apache-2.0', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2294, 'n_words': 535, 'path': 'a-proof-time-series/3-feature_engineering.ipynb', 'repo_name': 'cltl/a-proof-zonmw', 'size': '6687'}