<jupyter_start><jupyter_text>Sepsis Competition - BMEG400D Preprocessing By <NAME> Sepsis is a life-threatening condition that occurs when the body's response to infection exceeds typical inflammatory immune response, resulting in tissue and organ damage. As early prediction of sepsis is potentially life-saving, our goal is to predict sepsis four hours before the clinical identification of sepsis. This is the first of two notebooks - this notebook will examine the provided data, preprocess it  and provide complete training and test files as .csv.  Import Libraries<jupyter_code>import os
import pandas as pd
import numpy as np
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Load Training Files<jupyter_code>train_directory = "C:/Users/serge/Documents/UBC/BMEG_591A/competition/training_2020-11-11"
test_directory = "C:/Users/serge/Documents/UBC/BMEG_591A/competition/testing_2020-11-11"
def load_challenge_data(file):
    with open(file, 'r') as f:
        header = f.readline().strip()
        column_names = header.split(',')
        data = np.loadtxt(f, delimiter=',')

    # Ignore SepsisLabel column if present.
    if column_names[-1] == 'SepsisLabel':
        column_names = column_names[:-1]
        data = data[:, :-1]

    return data
# Find files.
train_files = []
for f in os.listdir(train_directory):
    if os.path.isfile(os.path.join(train_directory, f)) and not f.lower().startswith('.') and f.lower().endswith('csv'):
        train_files.append(f)
print(f"The number of training files is {len(train_files)}.")
for i in tqdm(range(len(train_files))):
    sos = pd.read_csv(train_directory + '/' + train_files[i])
    sos['WBC'] = sos['WBC'].fillna(np.mean(sos['WBC']))
    sos['Hgb'] = sos['Hgb'].fillna(np.mean(sos['Hgb']))
    sos['Hct'] = sos['Hct'].fillna(np.mean(sos['Hct']))
    sos['Potassium'] = sos['Potassium'].fillna(np.mean(sos['Potassium']))
    sos['Magnesium'] = sos['Magnesium'].fillna(np.mean(sos['Magnesium']))
    sos['Glucose'] = sos['Glucose'].fillna(np.mean(sos['Glucose']))
    sos['Creatinine'] = sos['Creatinine'].fillna(np.mean(sos['Creatinine']))
    sos['BUN'] = sos['BUN'].fillna(np.mean(sos['BUN']))
    sos['pH'] = sos['pH'].fillna(np.mean(sos['pH']))
    sos['FiO2'] = sos['FiO2'].fillna(np.mean(sos['FiO2']))
    sos['Resp'] = sos['Resp'].bfill()
    sos['DBP'] = sos['DBP'].bfill()
    sos['MAP'] = sos['MAP'].bfill()
    sos['SBP'] = sos['SBP'].bfill()
    sos['Temp'] = sos['Temp'].fillna(np.mean(sos['Temp']))
    sos['O2Sat'] = sos['O2Sat'].bfill()
    sos['HR'] = sos['HR'].bfill()
    sos['O2Sat'] = sos['O2Sat'].ffill()
    sos['HR'] = sos['HR'].ffill()
    sos['Resp'] = sos['Resp'].ffill()
    sos['DBP'] = sos['DBP'].ffill()
    sos['MAP'] = sos['MAP'].ffill()
    sos['SBP'] = sos['SBP'].ffill()
    if i == 0:
        het = sos
    else:
        try:
            het = pd.concat([het, sos]).reset_index()
        except:
            het.drop(columns='level_0', inplace=True)
            het = pd.concat([het, sos]).reset_index()
        
het.head()<jupyter_output><empty_output><jupyter_text>Load Testing Files<jupyter_code># Find files.
test_files = []
for f in os.listdir(test_directory):
    if os.path.isfile(os.path.join(test_directory, f)) and not f.lower().startswith('.') and f.lower().endswith('csv'):
        test_files.append(f)
print(f"The number of test files is {len(test_files)}.")
for i in tqdm(range(len(test_files))):
    sos = pd.read_csv(test_directory + '/' + test_files[i])
    sos['WBC'] = sos['WBC'].fillna(np.mean(sos['WBC']))
    sos['Hgb'] = sos['Hgb'].fillna(np.mean(sos['Hgb']))
    sos['Hct'] = sos['Hct'].fillna(np.mean(sos['Hct']))
    sos['Potassium'] = sos['Potassium'].fillna(np.mean(sos['Potassium']))
    sos['Magnesium'] = sos['Magnesium'].fillna(np.mean(sos['Magnesium']))
    sos['Glucose'] = sos['Glucose'].fillna(np.mean(sos['Glucose']))
    sos['Creatinine'] = sos['Creatinine'].fillna(np.mean(sos['Creatinine']))
    sos['BUN'] = sos['BUN'].fillna(np.mean(sos['BUN']))
    sos['pH'] = sos['pH'].fillna(np.mean(sos['pH']))
    sos['FiO2'] = sos['FiO2'].fillna(np.mean(sos['FiO2']))
    sos['Resp'] = sos['Resp'].bfill()
    sos['DBP'] = sos['DBP'].bfill()
    sos['MAP'] = sos['MAP'].bfill()
    sos['SBP'] = sos['SBP'].bfill()
    sos['Temp'] = sos['Temp'].fillna(np.mean(sos['Temp']))
    sos['O2Sat'] = sos['O2Sat'].bfill()
    sos['HR'] = sos['HR'].bfill()
    sos['O2Sat'] = sos['O2Sat'].ffill()
    sos['HR'] = sos['HR'].ffill()
    sos['Resp'] = sos['Resp'].ffill()
    sos['DBP'] = sos['DBP'].ffill()
    sos['MAP'] = sos['MAP'].ffill()
    sos['SBP'] = sos['SBP'].ffill()
    if i == 0:
        test = sos
    else:
        try:
            test = pd.concat([test, sos]).reset_index()
        except:
            test.drop(columns='level_0', inplace=True)
            test = pd.concat([test, sos]).reset_index()
        
test.drop(columns='level_0', inplace=True)
het.drop(columns='level_0', inplace=True)
combined = pd.concat([test, het]).reset_index()<jupyter_output><empty_output><jupyter_text>Get Global Means<jupyter_code>wbc_mean = np.mean(combined['WBC'])
hgb_mean = np.mean(combined['Hgb'])
hct_mean = np.mean(combined['Hct'])
pot_mean = np.mean(combined['Potassium'])
mag_mean = np.mean(combined['Magnesium'])
glu_mean = np.mean(combined['Glucose'])
crea_mean = np.mean(combined['Creatinine'])
bun_mean = np.mean(combined['BUN'])
ph_mean = np.mean(combined['pH'])
fio_mean = np.mean(combined['FiO2'])
dbp_mean = np.mean(combined['DBP'])
map_mean = np.mean(combined['MAP'])
sbp_mean = np.mean(combined['SBP'])
temp_mean = np.mean(combined['Temp'])
resp_mean = np.mean(combined['Resp'])
hr_mean = np.mean(combined['HR'])
ic_mean = np.mean(combined['ICULOS'])
o2_mean = np.mean(combined['O2Sat'])
print(ic_mean)
print(hr_mean)
print(o2_mean)
print(temp_mean)
print(sbp_mean)
print(map_mean)
print(dbp_mean)
print(resp_mean)
print(fio_mean)
print(ph_mean)
print(bun_mean)
print(crea_mean)
print(glu_mean)
print(mag_mean)
print(pot_mean)
print(hct_mean)
print(hgb_mean)
print(wbc_mean)
print('Part of missing values for every column')
print(combined.isnull().sum() / len(combined))<jupyter_output>Part of missing values for every column
level_0        0.000000
Age            0.000000
BUN            0.032326
Creatinine     0.032355
DBP            0.143948
FiO2           0.434664
Glucose        0.027118
HR             0.000000
Hct            0.037807
Hgb            0.038575
ICULOS         0.000000
MAP            0.001193
Magnesium      0.087222
O2Sat          0.000000
Potassium      0.032162
Resp           0.000983
SBP            0.005506
SepsisLabel    0.000000
Sex            0.000000
Temp           0.007001
WBC            0.042355
index          0.999446
pH             0.422031
dtype: float64<jupyter_text>Load the Training Data Again<jupyter_code>for i in tqdm(range(len(train_files))):
    sos = pd.read_csv(train_directory + '/' + train_files[i])
    if i == 0:
        het = sos
    else:
        try:
            het = pd.concat([het, sos]).reset_index()
        except:
            het.drop(columns='level_0', inplace=True)
            het = pd.concat([het, sos]).reset_index()     
het.sample(4)<jupyter_output><empty_output><jupyter_text>Load the Testing Data Again<jupyter_code>for i in tqdm(range(len(test_files))):
    sos = pd.read_csv(test_directory + '/' + test_files[i])

    if i == 0:
        test = sos
    else:
        try:
            test = pd.concat([test, sos]).reset_index()
        except:
            test.drop(columns='level_0', inplace=True)
            test = pd.concat([test, sos]).reset_index()
        
test.sample(4)<jupyter_output><empty_output><jupyter_text>Fill NaNs with Global Means<jupyter_code>het['WBC'] = het['WBC'].fillna(wbc_mean)
het['Hgb'] = het['Hgb'].fillna(hgb_mean)
het['Hct'] = het['Hct'].fillna(hct_mean)
het['Potassium'] = het['Potassium'].fillna(pot_mean)
het['Magnesium'] = het['Magnesium'].fillna(mag_mean)
het['Glucose'] = het['Glucose'].fillna(glu_mean)
het['Creatinine'] = het['Creatinine'].fillna(crea_mean)
het['BUN'] = het['BUN'].fillna(bun_mean)
het['pH'] = het['pH'].fillna(ph_mean)
het['FiO2'] = het['FiO2'].fillna(fio_mean)
het['DBP'] = het['DBP'].fillna(dbp_mean)
het['MAP'] = het['MAP'].fillna(map_mean)
het['SBP'] = het['SBP'].fillna(sbp_mean)
het['Temp'] = het['Temp'].fillna(temp_mean)
het['Resp'] = het['Resp'].fillna(resp_mean)
het['HR'] = het['Resp'].fillna(hr_mean)
het['O2Sat'] = het['O2Sat'].fillna(o2_mean)
het.sample(4)
test['WBC'] = test['WBC'].fillna(wbc_mean)
test['Hgb'] = test['Hgb'].fillna(hgb_mean)
test['Hct'] = test['Hct'].fillna(hct_mean)
test['Potassium'] = test['Potassium'].fillna(pot_mean)
test['Magnesium'] = test['Magnesium'].fillna(mag_mean)
test['Glucose'] = test['Glucose'].fillna(glu_mean)
test['Creatinine'] = test['Creatinine'].fillna(crea_mean)
test['BUN'] = test['BUN'].fillna(bun_mean)
test['pH'] = test['pH'].fillna(ph_mean)
test['FiO2'] = test['FiO2'].fillna(fio_mean)
test['DBP'] = test['DBP'].fillna(dbp_mean)
test['MAP'] = test['MAP'].fillna(map_mean)
test['SBP'] = test['SBP'].fillna(sbp_mean)
test['Temp'] = test['Temp'].fillna(temp_mean)
test['Resp'] = test['Resp'].fillna(resp_mean)
test['HR'] = test['Resp'].fillna(hr_mean)
test['O2Sat'] = test['O2Sat'].fillna(o2_mean)
test.sample(4)<jupyter_output><empty_output><jupyter_text>Save training.csv & testing.csv<jupyter_code>het.to_csv('training.csv', index=False)
test.to_csv('testing.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1967195767195767, 'fraction_numerical': 0.024973544973544974, 'mean_word_length': 4.348613469156763, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '383627', 'license': 'Apache-2.0', 'n_tokens_mistral': 3922, 'n_tokens_neox': 3657, 'n_words': 570, 'path': 'data_preprocessing.ipynb', 'repo_name': 'sergeiissaev/LGBM_competition_code', 'size': '9457'}