<jupyter_start><jupyter_text>Collect time-invariant data<jupyter_code>time_inv_features = listfile_df[['Icustay', 'Mortality', 'Age', 'Ethnicity', 'Gender']]
pickle.dump(time_inv_features, 
            open('../data/in_hospital_mortality_181111/time_inv_features.pkl', 'wb'))
time_inv_features = pickle.load(open('../data/in_hospital_mortality_181111/time_inv_features.pkl', 'rb'))
time_inv_features = time_inv_features.rename(index=str, columns={'Icustay': 'icustay_id'})
one_hot = pd.get_dummies(time_inv_features['Ethnicity'], prefix='Ethnicity')
time_inv_features = time_inv_features.drop('Ethnicity', axis=1).join(one_hot)
time_inv_features.head(5)
paitnet_info = pd.read_csv('/h/maimingj/projects/autodiagnosis/data/mortality_181108/elixhauser_ahrq.csv')
paitnet_info.head(5)
time_inv_features = time_inv_features.merge(paitnet_info, on='icustay_id', how='inner')
time_inv_features = time_inv_features.drop(['hadm_id', 'subject_id'], axis=1)
time_inv_features.head(5)
pickle.dump(time_inv_features, 
            open('../data/in_hospital_mortality_181111/time_inv_features.pkl', 'wb'))
time_inv_features.columns<jupyter_output><empty_output><jupyter_text>Collect time-series data<jupyter_code>import time
from datetime import datetime as dt

def get_diff_time_by_hours(time1, time2):
    '''
    time1: '2018-02-11 16:33:43'
    time2: '2018-02-12 16:33:43'
    '''
    # Handle '2123-04-13' does not match format '%Y-%m-%d %H:%M:%S'
    if len(time1) == 10:
        time1 += ' 00:00:00'
    if len(time2) == 10:
        time2 += ' 00:00:00'

    a = dt.strptime(time1, '%Y-%m-%d %H:%M:%S')
    b = dt.strptime(time2, '%Y-%m-%d %H:%M:%S')

    diff = b - a
    hours = diff.days * 24
    hours += diff.seconds / 3600.
    return hours

def get_real_end_time(list_df, i):
    # get real end hour == min(outtime - intime, deathtime - intime)
    row_i = list_df.iloc[i]
    intime, deathtime, outtime = row_i['Intime'], row_i['Deathtime'], row_i['Outtime']
    rel_end_time = get_diff_time_by_hours(intime, outtime)

    if pd.notnull(deathtime):
        rel_deathtime = get_diff_time_by_hours(intime, deathtime)
        rel_end_time = min(rel_deathtime, rel_end_time)
    
    return rel_end_time

def get_sparse_coding_from_dataset(list_df,
                                   data_dir,
                                   measurement_dirname,
                                   max_encounters=-1):
    """ Max encounter == -1 => use all encounters """
    start_time = time.time()
    Ts, Ys, ind_ts, ind_fs, icustay_ids, all_labels, all_rel_end_time = [], [], [], [], [], [], []
    counts = {}
    idx_to_feature_name = {}
    feature_name_to_idx = {}
    if max_encounters == -1:
        max_encounters = len(list_df)

    for i in range(max_encounters):
        T, Y, ind_t, ind_f = [], [], [], []
        
        rel_end_time = get_real_end_time(list_df, i)
        
        # Process time-series meaasurement
        row_i = list_df.iloc[i]
        fname = '{}/{}/{}_{}'.format(data_dir, measurement_dirname, row_i['Subject'], row_i['Episode'])
        measurements = pd.read_csv(fname)

        ## Filter measurement before ICU entry time and ICU outtime/deathtime
        measurements = measurements[(measurements['Hours'] >= 0) &
                                    (measurements['Hours'] <= rel_end_time)].reset_index(drop=True)

        T = measurements['Hours'].tolist()
        for j, j_name in enumerate(list(measurements)[2:]):
            valid_measurements = measurements[pd.notnull(measurements[j_name])]
            if len(valid_measurements) > 0:
                Y += valid_measurements[j_name].tolist()
                ind_f += [j] * len(valid_measurements)
                ind_t += valid_measurements.index.tolist()
                if j_name in counts:
                    counts[j_name] += len(valid_measurements)
                else:
                    counts[j_name] = len(valid_measurements)
            
            if i == 0:
                feature_name_to_idx[j_name] = j
                idx_to_feature_name[j] = j_name
        
        if len(ind_t) > 0:
            Ts.append(T)
            Ys.append(Y)
            ind_ts.append(ind_t)
            ind_fs.append(ind_f)
            icustay_ids.append(row_i['Icustay'])
            all_rel_end_time.append(rel_end_time)
            all_labels.append(row_i['Mortality'])

        # progress report
        if i % 999 == 0:
            print('Finished {} / {}. Took {}s'.format(
                i, max_encounters, time.time() - start_time))
            
    return Ts, Ys, ind_ts, ind_fs, \
           icustay_ids, all_labels, all_rel_end_time, counts, \
           feature_name_to_idx, idx_to_feature_name
Ts, Ys, ind_ts, ind_fs, \
icustay_ids, all_labels, all_rel_end_time, counts,\
feature_name_to_idx, idx_to_feature_name =\
    get_sparse_coding_from_dataset(list_df=listfile_df,
                                   data_dir=data_dir, 
                                   measurement_dirname=measurement_dirname, 
                                   max_encounters=-1)
counts
data_dict = {
    'icustay_ids': icustay_ids,
    'Ts': Ts,
    'Ys': Ys,
    'ind_ts': ind_ts,
    'ind_fs': ind_fs,
    'all_labels': all_labels,
    'all_rel_end_time': all_rel_end_time,
    'counts': counts,
    'feature_name_to_idx': feature_name_to_idx,
    'idx_to_feature_name': idx_to_feature_name
}

pickle.dump(data_dict, open('../data/in_hospital_mortality_181111/unnorm_time_var_features.pkl', 'wb'))
import pickle
data_dict = pickle.load(open('../data/in_hospital_mortality_181111/unnorm_time_var_features.pkl', 'rb'))
icustay_ids = data_dict['icustay_ids']
Ts = data_dict['Ts']
Ys = data_dict['Ys']
ind_ts = data_dict['ind_ts']
ind_fs = data_dict['ind_fs']
all_labels = data_dict['all_labels']
all_rel_end_time = data_dict['all_rel_end_time']
{k: len(data_dict[k]) for k in data_dict}<jupyter_output><empty_output><jupyter_text>Aggregate features<jupyter_code>del counts['Temperature (F)']
del idx_to_feature_name[feature_name_to_idx['Temperature (F)']]
del feature_name_to_idx['Temperature (F)']
aggregators = dict([(name, []) for name in counts])
# sum all the fields together. Get the summary statistics, then convert them again.
for Y, T, ind_f, ind_t in zip(Ys, Ts, ind_fs, ind_ts):
    for idx, (the_Y, the_ind_f) in enumerate(zip(Y, ind_f)):
        if the_Y < 0: # Should not have negative value
            continue
        
        if not the_ind_f in idx_to_feature_name:
            continue
        feature_name = idx_to_feature_name[the_ind_f]
        
        the_Y = np.log(0.1 + the_Y)
        
        aggregators[feature_name].append(the_Y)<jupyter_output><empty_output><jupyter_text>Filter low number measurement<jupyter_code>all_len = [len(aggregators[k]) for k in aggregators]
max(all_len) 
size_threshold = 0.001
selected_aggregators = {k: aggregators[k] for k in aggregators if len(aggregators[k]) >= max(all_len) * size_threshold}  
selected_idx_to_feature_name = {feature_name_to_idx[k]: k for k in selected_aggregators}
print('Number of measurements per features after filetering by number')

print({k: len(aggregators[k]) for k in selected_aggregators})
print('Number of features left: {}'.format(len(selected_idx_to_feature_name)))
size_threshold = 0.005
selected_aggregators = {k: aggregators[k] for k in aggregators if len(aggregators[k]) >= max(all_len) * size_threshold}  
selected_idx_to_feature_name = {feature_name_to_idx[k]: k for k in selected_aggregators}
print('Number of measurements per features after filetering by number')

print({k: len(aggregators[k]) for k in selected_aggregators})
print('Number of features left: {}'.format(len(selected_idx_to_feature_name)))
size_threshold = 0.01
selected_aggregators = {k: aggregators[k] for k in aggregators if len(aggregators[k]) >= max(all_len) * size_threshold}  
selected_idx_to_feature_name = {feature_name_to_idx[k]: k for k in selected_aggregators}
print('Number of measurements per features after filetering by number')

print({k: len(aggregators[k]) for k in selected_aggregators})
print('Number of features left: {}'.format(len(selected_idx_to_feature_name)))
for k in selected_aggregators:
    print(k, '&', len(selected_aggregators[k]), '&',
          np.round(len(selected_aggregators[k]) / max(all_len), 3), '\\\\')
##print({k: np.round(len(aggregators[k]) / max(all_len), 3)  for k in aggregators})
temp_lst = [len(aggregators[k]) for k in selected_aggregators]
temp_lst.sort()
temp_lst
size_threshold = 0.05
selected_aggregators = {k: aggregators[k] for k in aggregators if len(aggregators[k]) >= max(all_len) * size_threshold}  
selected_idx_to_feature_name = {feature_name_to_idx[k]: k for k in selected_aggregators}
print('Number of measurements per features after filetering by number')

print({k: len(aggregators[k]) for k in selected_aggregators})
print('Number of features left: {}'.format(len(selected_idx_to_feature_name)))
size_threshold = 0.1
selected_aggregators = {k: aggregators[k] for k in aggregators if len(aggregators[k]) >= max(all_len) * size_threshold}  
selected_idx_to_feature_name = {feature_name_to_idx[k]: k for k in selected_aggregators}
print('Number of measurements per features after filetering by number')

print({k: len(aggregators[k]) for k in selected_aggregators})
print('Number of features left: {}'.format(len(selected_idx_to_feature_name)))<jupyter_output>Number of measurements per features after filetering by number
{'Creatinine (urine)': 3929745, 'Fraction inspired oxygen': 627577, 'Glucose (urine)': 4204926, 'Magnesium': 3904218, 'Sodium': 3930865, 'Systolic blood pressure': 797435, 'Temperature (C)': 1116295}
Number of features left: 7<jupyter_text>Calculate stats for removing outliers and normalizing<jupyter_code>upperbound_dict, lowerbound_dict, mean_dict, std_dict = {}, {}, {}, {}

for k in selected_aggregators:
    tmp = np.array(selected_aggregators[k])
    
    IQR = np.percentile(tmp, 75) - np.percentile(tmp, 25)
    median = np.percentile(tmp, 50)
    
    upperbound_dict[k] = median + 4.5 * IQR
    lowerbound_dict[k] = median - 4.5 * IQR
    
    tmp = tmp[tmp < upperbound_dict[k]]
    tmp = tmp[tmp > lowerbound_dict[k]]
    
    mean_dict[k] = np.mean(tmp)
    std_dict[k] = np.std(tmp)<jupyter_output><empty_output><jupyter_text>Visualize historgram of feature after removing outlier<jupyter_code>from scipy.stats import kurtosistest
for k in selected_aggregators:
    tmp = np.array(selected_aggregators[k])
    tmp = tmp[tmp >= lowerbound_dict[k]]
    tmp = tmp[tmp <= upperbound_dict[k]]
    print(k, kurtosistest(tmp))
for k in selected_aggregators:
    tmp = np.array(selected_aggregators[k])
    tmp = tmp[tmp >= lowerbound_dict[k]]
    tmp = tmp[tmp <= upperbound_dict[k]]
    plot_hist_simple(tmp, title='Histogram of {}'.format(k), xlabel='value', ylabel='count')
    
#     if np.sum(tmp < 0) != 0:
#         print('%s has negative value!')

#     plot_hist_simple(np.log(0.1 + tmp[tmp > 0]), 
#                      title='Histogram of {} after log-transform'.format(k),
#                      xlabel='log-value', ylabel='count')
selected_idx_to_feature_name
selected_feature_name_to_idx = {selected_idx_to_feature_name[k]: i  
                                for i, k in enumerate(selected_idx_to_feature_name)}
selected_feature_name_to_idx<jupyter_output><empty_output><jupyter_text>Normalize features<jupyter_code>def normalize(Ys, ind_fs, ind_ts):
    new_Ys, new_ind_fs, new_ind_ts = [], [], []

    for Y, ind_f, ind_t in zip(Ys, ind_fs, ind_ts):
        new_Y, new_ind_f, new_ind_t = [], [], []
        
        for idx, (theY, the_ind_f, the_ind_t) in enumerate(zip(Y, ind_f, ind_t)):
            if not the_ind_f in selected_idx_to_feature_name:
                continue
            elif theY < 0:
                continue
                
            feature_name = selected_idx_to_feature_name[the_ind_f]

            theY = np.log(0.1 + theY)
                
            
            if theY < lowerbound_dict[feature_name] or theY > upperbound_dict[feature_name]:
                continue

            theY = (theY - mean_dict[feature_name]) / std_dict[feature_name]
            
            new_Y.append(theY)
            new_ind_f.append(selected_feature_name_to_idx[feature_name])
            new_ind_t.append(the_ind_t)
            
        new_Ys.append(new_Y)
        new_ind_fs.append(new_ind_f)
        new_ind_ts.append(new_ind_t)
        
    return new_Ys, new_ind_fs, new_ind_ts
norm_Ys, norm_ind_fs, norm_ind_ts = normalize(Ys, ind_fs, ind_ts)
data_dict = {
    'icustay_ids': icustay_ids,
    'Ts': Ts,
    'Ys': norm_Ys,
    'ind_ts': norm_ind_ts,
    'ind_fs': norm_ind_fs,
    'all_labels': all_labels,
    'all_rel_end_time': all_rel_end_time,
    'selected_feature_name_to_idx': selected_feature_name_to_idx,
    'selected_idx_to_feature_name': selected_idx_to_feature_name
}

pickle.dump(data_dict, open('../data/in_hospital_mortality_181111/norm_time_var_features.pkl', 'wb'))
len(data_dict['Ts'][1])
max(data_dict['ind_ts'][1])
selected_feature_name_to_idx
selected_idx_to_feature_name<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12148249828414551, 'fraction_numerical': 0.016548463356973995, 'mean_word_length': 3.5773123909249565, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '286168', 'license': 'MIT', 'n_tokens_mistral': 4832, 'n_tokens_neox': 4572, 'n_words': 864, 'path': 'notebooks/1108 - process_MIMIC_data.ipynb', 'repo_name': 'zzzace2000/mimic-preprocess', 'size': '13113'}