<jupyter_start><jupyter_text>for i in range(len(data['Sex'].values)):    if data['Sex'].values[i] =='Male':        data['Sex'].values[i] = 0    else:        data['Sex'].values[i] = 1for i in range(len(data['SmokingStatus'].values)):    if data['SmokingStatus'].values[i] =='Ex-smoker':        data['SmokingStatus'].values[i] = 0    elif data['SmokingStatus'].values[i] =='Never smoked':        data['SmokingStatus'].values[i] = 1         else:        data['SmokingStatus'].values[i] = 2<jupyter_code>data
x_cols = ['Weeks','Base_Week','Base_FVC','Age','sex_m','sex_f','sm_es','sm_ns','sm_cs']
x_train = data[x_cols].loc[data['Type'] == "train"].values.astype(np.float)
y_train = data[prediction_col].loc[data['Type'] == "train"].values.astype(np.float)
x_test = data[x_cols].loc[data['Type'] == "test"].values.astype(np.float)
x_train = x_train.astype(np.float32)
y_train = y_train.astype(np.float32)
x_test = x_test.astype(np.float32)
x_train.shape,y_train.shape,x_test.shape
type(x_train[0][3])
type(x_train),type(y_train),type(x_test)
data
class Data_Generator(tf.keras.utils.Sequence):
    
    def __init__(self,batch_size,patient_ids,tab_data,dim,target=None,train=True,augment=False):
        self.batch_size = batch_size
        self.image_ids = patient_ids
        self.augment = augment
        self.dim = dim
        self.target = target
        self.indices = range(len(self.image_ids))
        self.train = train
        self.tab_data = tab_data
        #self.on_epoch_end()
    
    def getimage(self,image_id):
        X1 = np.zeros((NUM_IMAGES,self.dim[1],self.dim[2], 1))
        if self.train:
            path = TRAIN_PATH
        else:
            path = TEST_PATH
        im_num = len(os.listdir(os.path.join(path,image_id)))
        if im_num < NUM_IMAGES+1:
            for i,dcm_i in enumerate(os.listdir(os.path.join(path,image_id))):
                try:
                    im = pydicom.dcmread(os.path.join(TRAIN_PATH,f'{image_id}/{dcm_i}'))
                    img = im.pixel_array/255
                    img = cv2.resize(img, (self.dim[1],self.dim[2]))
                    img = np.reshape(img,(IMAGE_DIM[1],IMAGE_DIM[2],1))
                    X1[i,] = img
                    if i>=NUM_IMAGES-1:
                        break
                except:
                    continue
        else:
            val = (im_num - NUM_IMAGES)//2
            dir_list = os.listdir(os.path.join(path,image_id))
            dir_list.sort()
            for i,dcm_i in enumerate(dir_list[val:]):
                try:
                    im = pydicom.dcmread(os.path.join(TRAIN_PATH,f'{image_id}/{dcm_i}'))
                    img = im.pixel_array/255
                    img = cv2.resize(img, (self.dim[1],self.dim[2]))
                    img = np.reshape(img,(IMAGE_DIM[1],IMAGE_DIM[2],1))
                    X1[i,] = img
                    if i>=NUM_IMAGES-1:
                        break
                except:
                    continue
               
        if self.augment == True:
            img = self.ImageAugment(img)
            return img
        return X1
    
    def on_epoch_end(self):
        return self.indices
    
    def getdata(self, image_id_list):
        X = np.empty((self.batch_size,*self.dim, 1))
        for i, im_id in enumerate(image_id_list):
            #print(i)
            X[i,] = self.getimage(im_id)
        
        return X
    '''
    def ImageAugment(self,image):
        augmentor = ImageAugmentor(image,axis_point=[self.dim/2,self.dim/2])
        augmentor.cutmix()
        #augmentor.zoom()
        augmentor.flip()
        augmentor.rotate()
        return augmentor.get_image()
    ''' 
    
    def __getitem__(self,index):
        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]
        
        image_id_list = [self.image_ids[k] for k in indices]
        tab_X = np.array([self.tab_data[k] for k in indices]).astype(np.float32)
        X = self.getdata(image_id_list)
        if self.train == True:
            target_list = [self.target[k] for k in indices]
            y = np.array(target_list).astype(np.float32)
            return X,y
        return X
    
    def __len__(self):
        return int(np.floor(len(self.indices)/self.batch_size))
    
C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype="float32")
#=============================#
def score(y_true, y_pred):
    tf.dtypes.cast(y_true, tf.float32)
    tf.dtypes.cast(y_pred, tf.float32)
    sigma = y_pred[:,2]-y_pred[:,0]
    fvc_pred = y_pred[:,1]
    #sigma_clip = sigma + C1
    sigma_clip = tf.maximum(sigma, C1)
    delta = tf.abs(y_true[:,0] - fvc_pred)
    delta = tf.minimum(delta, C2)
    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )
    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)
    return K.mean(metric)
#============================#
def qloss(y_true, y_pred):
    # Pinball loss for multiple quantiles
    qs = [0.2,0.50,0.8]
    q = tf.constant(np.array([qs]), dtype=tf.float32)
    e = y_true - y_pred
    v = tf.maximum(q*e,(q-1)*e)
    return K.mean(v)
#=============================#
def mloss(_lambda):
    def loss(y_true, y_pred):
        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)
    return loss
#=================
def build_model_dense():
    
    inp = L.Input((9,))
    x = L.Dense(128,activation ='relu',kernel_regularizer=R.l2(5e-4),name='dense')(inp)
    x = L.Dense(128,activation ='relu',kernel_regularizer=R.l2(5e-4),name='dense_1')(x)
    x = L.Dense(64,activation ='relu',kernel_regularizer=R.l2(5e-5),name='dense_2')(x)
    o1 = L.Dense(3,activation = 'linear',name='dense_3')(x)
    o2 = L.Dense(3,activation = 'relu',name='dense_4')(x)
    pred1 = L.Lambda(lambda x: (x[0]+(tf.cumsum(x[1],axis=1))))([o1,o2])    
    
    
    model = M.Model(inputs=inp,outputs=pred1)
    
    model.compile(loss=mloss(0.85), optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), metrics=[score])
    
    return model
dense_model = build_model_dense()
dense_model.summary()
from sklearn.model_selection import KFold
folds = 5
KF = KFold(n_splits=folds,shuffle=True,random_state=24)
%%time
counter=0
stopper = tf.keras.callbacks.EarlyStopping(monitor='loss',mode='min',patience=100,restore_best_weights=True)
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1,patience=50, min_lr=1e-3)
for tr_idx, val_idx in KF.split(x_train):
    counter+=1
    print(f'############## FOLD {counter} ###############')
    history = dense_model.fit(x_train[tr_idx],y_train[tr_idx],epochs=500,batch_size = 256,verbose=0,validation_data=(x_train[val_idx],y_train[val_idx]))
    print("train", dense_model.evaluate(x_train[tr_idx], y_train[tr_idx], verbose=0, batch_size=256))
    print("val", dense_model.evaluate(x_train[val_idx], y_train[val_idx], verbose=0, batch_size=256))
dense_model.save('dense_model.h5')
dense_model.summary()
def build_model():
        
    inp2 = L.Input(shape=(*IMAGE_DIM,1,),name='conv_input')
    
    x = L.Conv3D(64,5,activation='relu')(inp2)
    
    x = L.Conv3D(64,5,activation='relu')(x)
    
    x = L.BatchNormalization()(x)
    
    x = L.MaxPooling3D()(x)
    
    x1 = L.Dropout(0.3)(x)
    
    x = L.Conv3D(64,7,activation='relu',padding='same')(x1)
    
    x = L.Dropout(0.3)(x)
    
    x = L.add([x,x1])
    
    x = L.Conv3D(256,7,activation='relu')(x)
    
    x = L.Conv3D(128,9,activation='relu')(x)
    
    x = L.BatchNormalization()(x)
    
    x = L.MaxPooling3D()(x)
    
    x2 = L.Dropout(0.43)(x)
    
    x = L.Conv3D(128,7,activation='relu',padding='same',data_format='channels_last')(x2)
    
    x = L.Dropout(0.43)(x)
    
    x = L.add([x,x2])
    
    x = L.BatchNormalization()(x)
    
    x = L.MaxPooling3D()(x)
    
    x = L.Dropout(0.43)(x)
    
    op1 = L.GlobalAveragePooling3D()(x)
    '''
    tab_model = M.load_model('dense_model.h5',custom_objects={'loss':mloss,'score':score})
    
    tab_model.trainable = False
    
    inp = L.Input((9,),name='input_d')

    d = tab_model.get_layer(name='dense')(inp)
    
    d1 = tab_model.get_layer(name='dense_1')(d)
    
    op2 = tab_model.get_layer(name='dense_2')(d1)    
    
    inp = L.Input(shape=(4,),name='tab_input')
    
    x = L.Dense(128,activation ='relu',kernel_regularizer=R.l2(5e-4))(inp)
    
    x = L.Dense(128,activation ='relu',kernel_regularizer=R.l2(4e-4))(x)
    
    op2 = L.Dense(64,activation ='relu',kernel_regularizer=R.l2(2e-5))(x)
 
    x = L.add([op1,op2])
    '''
    o1 = L.Dense(3,activation = 'linear',name = 'dense_f1')(op1)
    o2 = L.Dense(3,activation = 'relu', name='dense_f2')(op1)
    
    pred1 = L.Lambda(lambda x: (x[0]+(tf.cumsum(x[1],axis=1))),name='output')([o1,o2])    
    
    
    model = M.Model(inputs=inp2,outputs=pred1)
    
    model.compile(loss=mloss(0.9), optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4), metrics=[score])
    
    return model

model = build_model()
model.summary()
folds = 5
KF = KFold(n_splits=folds,shuffle=True,random_state=24)
x_train = data[x_cols].loc[data['Type'] == "train"].values
y_train = data[prediction_col].loc[data['Type'] == "train"].values
x_test = data[x_cols].loc[data['Type'] == "test"].values
def get_lr_callback():
    lr_start   = 0.001
    lr_max     = 0.001
    lr_min     = 0.00001
    lr_ramp_ep = EPOCHS
    lr_sus_ep  = 0
    lr_decay   = 0.9
   
    def lrfn(epoch):
        if epoch < lr_ramp_ep:
            lr = lr_start - ( lr_start - lr_min ) / lr_ramp_ep * epoch 
            
        elif epoch < lr_ramp_ep + lr_sus_ep:
            lr = lr_min
            '''
        else:
            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min
            '''
        return lr
    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
    return lr_callback
#tf.keras.utils.plot_model(model, "multi_input_and_output_model.png", show_shapes=True)

%%time
x_train = data[x_cols].loc[data['Type'] == "train"].values
patient_ids = data['Patient'].loc[data['Type']=='train'].values
target_val = data[prediction_col].loc[data['Type'] == "train"].values
for fold,(t_idx,v_idx) in enumerate(KF.split(patient_ids)):
    print(f"################ FOLD {fold+1} #####################")
    train_gen = Data_Generator(BATCH_SIZE,patient_ids[t_idx],x_train[t_idx],IMAGE_DIM,target=target_val[t_idx],train=True)
    val_gen = Data_Generator(BATCH_SIZE,patient_ids[v_idx],x_train[v_idx],IMAGE_DIM,target=target_val[v_idx],train=True)
    total_val_train = len(patient_ids[v_idx])
    total_train = len(patient_ids[t_idx])
    history = model.fit(
        train_gen,
        steps_per_epoch=total_train//BATCH_SIZE,
        epochs=EPOCHS,
        validation_data = val_gen,
        validation_steps = total_val_train//BATCH_SIZE,
        verbose=1,
        callbacks=[get_lr_callback()]
    )
    
    plt.figure()
    plt.plot(list(range(EPOCHS)),history.history['val_loss'])
    plt.plot(list(range(EPOCHS)),history.history['loss'])
    plt.xlabel("Epochs")
    plt.ylabel('loss')
    plt.legend(['val_loss','loss'])
    plt.show()
    
    plt.figure()
    plt.plot(list(range(EPOCHS)),history.history['val_score'])
    plt.plot(list(range(EPOCHS)),history.history['score'])
    plt.xlabel("Epochs")
    plt.ylabel('score')
    plt.legend(['val_score','score'])
    plt.show()
model.save('model.h5')
#model = M.load_model('../input/tab-data-osic/model.h5',custom_objects={'loss':mloss,'score':score})
#model.summary()
#test_patient_ids = data['Patient'].loc[data['Type'] == "test"].values
#test_gen = Data_Generator(2,test_patient_ids,x_test,IMAGE_DIM,train=False)
#print("Inferencing")
#pred = model.predict(test_gen,verbose=1,batch_size=BATCH_SIZE)3
#pred
#conf = pred[:,2] - pred[:,0]
#for i in range(len(conf)):
#    conf[i] = max(conf[i],70)
#pred[:,1]
#pred_dict = {'FVC':pred[:,1],'Confidence':conf}
#pred_df = pd.DataFrame(pred_dict)
#sub['Confidence'] = pred_df['Confidence']
#sub['FVC'] = pred_df['FVC']
#subm = sub[['Patient_Week','FVC','Confidence']].copy()
#subm.to_csv("submission.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1786629204198, 'fraction_numerical': 0.024212874969010825, 'mean_word_length': 3.112130479102956, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '435362', 'license': 'Apache-2.0', 'n_tokens_mistral': 4818, 'n_tokens_neox': 4632, 'n_words': 597, 'path': 'tabular-and-3d-data-training.ipynb', 'repo_name': 'SamsLogic/OSIC-pulmonary-fibrosis-kaggle-competition-code-527-2097', 'size': '12101'}