<jupyter_start><jupyter_text>Google Brain - Ventilator Pressure Prediction Competition Training setup, without inference<jupyter_code>%config Completer.use_jedi = False
import numpy as np
import pandas as pd
import seaborn as sns

import matplotlib.pyplot as plt
from matplotlib.pyplot import imshow, imread

from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import mean_absolute_error

import gc
R_SEED = 42
submission_ex = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/sample_submission.csv')
train_data = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/train.csv')
# test_data = pd.read_csv('/kaggle/input/ventilator-pressure-prediction/test.csv')
gc.collect()
target_data = train_data[['pressure']].copy()
train_id_data = train_data[['id']].copy()
# test_id_data = test_data[['id']].copy()
train_data.drop(['id', 'pressure'], axis=1, inplace=True) 
# test_data.drop(['id'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Columns* ```id``` - globally-unique time step identifier across an entire file* ```breath_id``` - **globally-unique time step for breaths*** ```R``` - lung attribute indicating how restricted the airway is (in cmH2O/L/S). Physically, this is the change in pressure per change in flow (air volume per time). Intuitively, one can imagine blowing up a balloon through a straw. We can change R by changing the diameter of the straw, with higher R being harder to blow.* ```C``` - lung attribute indicating how compliant the lung is (in mL/cmH2O). Physically, this is the change in volume per change in pressure. Intuitively, one can imagine the same balloon example. We can change C by changing the thickness of the balloon’s latex, with higher C having thinner latex and easier to blow.* ```time_step``` - the actual time stamp.* ```u_in``` - the control input for the inspiratory solenoid valve. Ranges from 0 to 100. Continuous variable from 0 to 100 representing the percentage the inspiratory solenoid valve is open to let air into the lung (i.e., 0 is completely closed and no air is let in and 100 is completely open)* ```u_out``` - the control input for the exploratory solenoid valve. Either 0 or 1. Control input is a binary variable representing whether the exploratory valve is open (1) or closed (0) to let air out.* ```pressure``` - the airway pressure measured in the respiratory circuit, measured in cmH2O.<jupyter_code>train_data.info()
target_data.info()
unique_train_breath_id = np.unique(train_data['breath_id'])
# unique_test_breath_id = np.unique(test_data['breath_id'])

print('unique_train_breath_id: ', unique_train_breath_id)
print('len: ', len(unique_train_breath_id))
# print('unique_test_breath_id: ', unique_test_breath_id)
# print('len: ', len(unique_test_breath_id))
assert all(e == 80 for e in [train_data['breath_id'].value_counts().max(),
                             train_data['breath_id'].value_counts().min()
                            #  test_data['breath_id'].value_counts().max(),
                            #  test_data['breath_id'].value_counts().min()
                            ])

n_of_cols = 3
n_of_rows = 2

fig, axes = plt.subplots(nrows = n_of_rows, ncols = n_of_cols, figsize=(25, 15))

i = 1
for c in train_data.columns:
    
    ax = axes[(i-1) // n_of_cols, (i-1) % n_of_cols]
    
    ax.set_title('feature: ' + c)
    sns.histplot(train_data[c].values, kde=False, color = 'k', bins=100, alpha=0.5, ax=ax, label='train')
    # sns.histplot(test_data[c].values, kde=False, color = 'orange', bins=100, alpha=0.5, ax=ax, label='test')
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.legend()
    
    i += 1

plt.legend()
plt.show()
fig = plt.figure(figsize = (20, 10))
ax = fig.gca()

ax.set_title('Target Value')
sns.histplot(target_data['pressure'].values, kde=False, color = 'k', bins=1000, alpha=0.5, ax=ax, label='Target Value')
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>FE [start]<jupyter_code>TARGET_MIN = target_data['pressure'].values.min()
TARGET_MAX = target_data['pressure'].values.max()
for (df, _id) in [(train_data, train_id_data)]: #, (test_data, test_id_data)]:
    ###########
    # ADDING FEATURES
    ###########
    
    #----------
    df['time_step_sin'] = np.abs(np.sin(_id * (2. * np.pi / 160)))
    #----------
    df['real_time_step_2'] = df['time_step'] - df.groupby('breath_id')['time_step'].shift(1).fillna(0)
    #----------
    uout = df['u_out'].values
    time_uout_0 = np.zeros(len(uout))
    time_uout_1 = np.zeros(len(uout))
    curr = uout[0]
    i_start = 0
    i_end = 0
    for idx, n in enumerate(uout):
        if curr != n:
            d = i_end - i_start
            for i in range(i_start, i_end + 1):
                if curr == 0:
                    time_uout_0[i] = np.sin((i - i_start) * (2. * np.pi / (2 * d))) + 0.1
                if curr == 1:
                    time_uout_1[i] = np.sin((i - i_start) * (2. * np.pi / (2 * d))) + 0.1
            i_start = idx
            i_end = idx
            curr = uout[idx]
        else:
            i_end = idx
    df['time_uout_0'] = time_uout_0
    df['time_uout_1'] = time_uout_1
    del uout, time_uout_0, time_uout_1
    #----------
    df['u_in_cumsum'] = (df['u_in']).groupby(df['breath_id']).cumsum()
    #----------
    df['u_in_lag_1'] = df.groupby('breath_id')['u_in'].shift(1).fillna(0)
    df['u_in_lag_n1'] = df.groupby('breath_id')['u_in'].shift(-1).fillna(0)
    df['u_in_lag_3'] = df.groupby('breath_id')['u_in'].shift(3).fillna(0)
    df['u_in_lag_n3'] = df.groupby('breath_id')['u_in'].shift(-3).fillna(0)
    #----------
    df['u_in_diff_1'] = df['u_in'] - df['u_in_lag_1']
    df['u_in_diff_n1'] = df['u_in_lag_n1'] - df['u_in']
    df['u_in_diff_3'] = df['u_in'] - df['u_in_lag_3']
    df['u_in_diff_n3'] = df['u_in_lag_n3'] - df['u_in']
    #----------
    df['bar_area'] = df['real_time_step_2'] * df['u_in_lag_1'] # for previous bar
    #----------
    df['area'] = df['time_step'] * df['u_in_lag_1']
    df['area'] = df.groupby('breath_id')['area'].cumsum()
    #----------
    df['pp'] = df['area'] / df['C']
    #----------
    df['vv'] = df['pp'] / df['R']
    #----------
    df['R_times_C'] = df['R'] * df['C'] / 1000
    #----------
    df['u_in_diff_max'] = df.groupby(['breath_id'])['u_in'].transform('max') - df['u_in']
    #----------
    df['cross'] = df['u_in'] * df['u_out']
    #----------
    df['cross2'] = df['time_step'] * df['u_out']
    #----------
    df['u_out_group_sum'] = df['u_out'].groupby(df['breath_id']).transform('sum')
    #----------
    df['u_in_sum_u_out0'] = ((df['u_in']).groupby(df['breath_id'] + df['u_out'] * 0.5)).transform('sum') * (df['u_out'] == 0).astype(int)
    #----------
    df['u_in_sum_u_out1'] = ((df['u_in']).groupby(df['breath_id'] + df['u_out'] * 0.5)).transform('sum') * (df['u_out'] == 1).astype(int)
    #----------
    df['one'] = 1
    df['count'] = (df['one']).groupby(df['breath_id']).cumsum()
    df['u_in_cum_mean'] = df['u_in_cumsum'] / df['count'] # --------------------- f
    #----------
    df['bpm'] = 1 / df['time_step'].groupby(df['breath_id']).transform('max')
    #----------
    uin = df['u_in'].values
    u_in_cum_f1 = np.zeros(len(uin))
    u_in_cum_f2 = np.zeros(len(uin))
    for i in range(int(len(uin) / 80)):
        curr_cum = 0
        count_for_f2 = 0
        for j in range(i*80, (i+1)*80):
            if uin[j] != 0:
                curr_cum += uin[j]
                count_for_f2 += 1
                u_in_cum_f1[j] = curr_cum
                u_in_cum_f2[j] = curr_cum / count_for_f2
            else:
                curr_cum = 0
                count_for_f2 = 0
    df['u_in_cum_f1'] = u_in_cum_f1
    df['u_in_cum_f2'] = u_in_cum_f2
    del uin, u_in_cum_f1, u_in_cum_f2
    #----------
    df['f_1'] = df['R'] * (df['u_out'] == 0).astype(int) * df['u_in']
    df['f_2'] = df['u_in'] * (df['u_out'] == 0).astype(int) / df['C']
    df['f_3'] = df['f_2'] / df['R']
    #----------
    
    
    
    ###########
    # DROP FEATURES
    ###########
    
    df.drop(['breath_id'], axis=1, inplace=True)
    df.drop(['one','count'], axis=1, inplace=True)

gc.collect()
# train

train_data['RC'] = (train_data['R'] ** 2).astype(str) + train_data['C'].astype(str)
train_data['R'] = train_data['R'].astype(str)
train_data['C'] = train_data['C'].astype(str)
train_data = pd.get_dummies(train_data)



# test

# test_data['RC'] = (test_data['R'] ** 2).astype(str) + test_data['C'].astype(str)
# test_data['R'] = test_data['R'].astype(str)
# test_data['C'] = test_data['C'].astype(str)
# test_data = pd.get_dummies(test_data)
gc.collect()
# FE [end]
scaler = RobustScaler() # MinMaxScaler RobustScaler StandardScaler
train_data = scaler.fit_transform(train_data)
# test_data = scaler.transform(test_data)
train_data = np.float32(train_data)
# test_data = np.float32(test_data)
gc.collect()
frame_len = 80
train_data = train_data.reshape(-1, frame_len, train_data.shape[1])
# test_data = test_data.reshape(-1, frame_len, test_data.shape[1])
target_data = target_data.to_numpy().reshape(-1, frame_len)
train_data.shape
target_data.shape
train_data, X_test, target_data, y_test = train_test_split(train_data, target_data, test_size=0.2, random_state=R_SEED)
gc.collect()
import tensorflow as tf
from tensorflow.keras import datasets, layers, models, Model
from tensorflow.keras.callbacks import LearningRateScheduler, ReduceLROnPlateau, EarlyStopping
from tensorflow.keras import backend as K
from tensorflow.keras.optimizers.schedules import ExponentialDecay
gc.collect()
def ANN(input_shape=train_data.shape[-2:]):
    
    x_input = layers.Input(shape=input_shape)
    
    C = layers.Conv2D(filters = 64, kernel_size = (7, 7), strides = (1, 1), activation='relu')(tf.expand_dims(x_input, -1))
    C = layers.Conv2D(filters = 128, kernel_size = (7, 7), strides = (1, 1), activation='relu')(C)
    C = layers.AveragePooling2D(pool_size=(5, 5), strides=(5,5), padding="same")(C)
    C = layers.Flatten()(C)
    C = layers.Dense(32, activation='selu')(C)
    C = layers.RepeatVector(80)(C)
    
    CC = layers.TimeDistributed(layers.Conv1D(filters=64, kernel_size=7, activation='relu'))(tf.expand_dims(x_input, -1))
    CC = layers.TimeDistributed(layers.Conv1D(filters=128, kernel_size=7, activation='relu'))(CC)
    CC = layers.TimeDistributed(layers.AveragePooling1D(pool_size=5, strides=5, padding="same"))(CC)
    CC = layers.TimeDistributed(layers.Flatten())(CC)
    CC = layers.TimeDistributed(layers.Dense(32, activation='selu'))(CC)
        
    X = layers.Bidirectional(layers.LSTM(512, return_sequences=True))(x_input) # --------------------------------------------------------
    X = layers.Bidirectional(layers.LSTM(384, return_sequences=True))(X)
    X = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(X)
    X = layers.Bidirectional(layers.LSTM(128, return_sequences=True))(X)
    
    X = layers.Concatenate()([X, C, CC])
    
    X = layers.Dense(128, activation='selu')(X)
    X = layers.Dense(1)(X)
    
    model = models.Model(inputs = x_input, outputs = X, name='GB-21')
    return model
print(TARGET_MIN, TARGET_MAX)
lr = ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=15, verbose=1)
es = EarlyStopping(monitor="val_loss", patience=30, verbose=1, mode="min", restore_best_weights=True)
model = ANN()
model.summary()<jupyter_output>Model: "GB-21"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_5 (InputLayer)            [(None, 80, 49)]     0                                            
__________________________________________________________________________________________________
tf.expand_dims_8 (TFOpLambda)   (None, 80, 49, 1)    0           input_5[0][0]                    
__________________________________________________________________________________________________
conv2d_8 (Conv2D)               (None, 74, 43, 64)   3200        tf.expand_dims_8[0][0]           
__________________________________________________________________________________________________
tf.expand_dims_9 (TFOpLambda)   (None, 80, 49, 1)    0           input_5[0][0]                [...]<jupyter_text>TPU<jupyter_code>gc.collect()
EPOCH = 300
BATCH_SIZE = 256 # with tpu we can go easily with 1024 even 2048, but with smaller batch size accuracy is way better

k = 5

test_preds = []

# detect and init the TPU
tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()

# instantiate a distribution strategy
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)

with tpu_strategy.scope():
    
    kfolds = KFold(n_splits = k, shuffle = True, random_state = R_SEED)
    curr_k = 1
    for train_index, val_index in kfolds.split(train_data):
        print('fold: ', curr_k)

        X_train, X_val = train_data[train_index], train_data[val_index]
        y_train, y_val = target_data[train_index], target_data[val_index]

        model = ANN()
        model.compile(
                optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001),
                loss = 'mae',
                metrics = ['mae'])

        history = model.fit(
            X_train,
            y_train,
            epochs = EPOCH,
            shuffle = True,
            validation_data = (X_val, y_val),
            batch_size = BATCH_SIZE,
            callbacks=[lr, es]
        )

        test_preds.append(model.predict(X_test, batch_size=BATCH_SIZE).squeeze().reshape(-1, 1).squeeze())

        model.save('model_'+str(curr_k)+'.h5',save_format='h5')
        
        curr_k += 1
        del model
        gc.collect()

    test_preds = sum(test_preds) / k
    mae = mean_absolute_error(y_test.reshape(-1, 1).squeeze(), test_preds)
    print(mae)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18004384414114985, 'fraction_numerical': 0.022063503288310585, 'mean_word_length': 3.295868772782503, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '433585', 'license': 'Apache-2.0', 'n_tokens_mistral': 5266, 'n_tokens_neox': 4990, 'n_words': 959, 'path': '[005_1]_ventilator_pressure_prediction_gb_2021.ipynb', 'repo_name': 'smf-9000/kaggle', 'size': '14141'}