<jupyter_start><jupyter_text>Model 1<jupyter_code>in_vars_RH = ['RH','TBP','CLDLIQBP','CLDICEBP','PS', 'SOLIN', 'SHFLX', 'LHFLX']
BASE_DIR = '/DFS-L/DATA/pritchard/ankitesg/'
in_vars = ['QBP','TBP','CLDLIQBP','CLDICEBP','PS', 'SOLIN', 'SHFLX', 'LHFLX']
out_vars = ['QBCTEND','TBCTEND']
TRAINFILE_RH = 'RG_RH_M4K_NORM_train_shuffle.nc'
NORMFILE_RH = 'RG_RH_M4K_NORM_norm.nc'
VALIDFILE_RH = 'RG_RH_M4K_NORM_valid.nc'
train_gen_RH = DataGenerator(
    data_fn = f"{BASE_DIR}datav4/{TRAINFILE_RH}",
    input_vars = in_vars_RH,
    output_vars = out_vars,
    norm_fn = f"{BASE_DIR}datav4/{NORMFILE_RH}",
    input_transform = ('mean', 'maxrs'),
    output_transform = scale_dict,
    batch_size=1024,
    shuffle=True,
    normalize_flag=True
)
train_gen_RH[0][0].shape
TRAINFILE = 'RG_SP_M4K_train_shuffle.nc'
NORMFILE = 'RG_SP_M4K_NORM_norm.nc'
VALIDFILE = 'RG_SP_M4K_valid.nc'
train_gen = DataGeneratorClimInvRealGeo(
    data_fn = f'{DATA_DIR}{TRAINFILE}',
    input_vars = in_vars,
    output_vars = out_vars,
    norm_fn = f'{DATA_DIR}{NORMFILE}',
    input_transform = ('mean', 'maxrs'),
    output_transform = scale_dict,
    batch_size=1024,
    shuffle=True,
    normalize_flag=True,
    lev=lev,
    hyam=hyam,hybm=hybm,
    rh_trans = True,t2tns_trans=False,
    lhflx_trans=False,
    scaling=False,
    interpolate=False,
    inp_subRH=train_gen_RH.input_transform.sub,inp_divRH=train_gen_RH.input_transform.div
)

train_gen[0][0].shape
valid_gen = DataGeneratorClimInvRealGeo(
    data_fn = f'{DATA_DIR}{VALIDFILE}',
    input_vars = in_vars,
    output_vars = out_vars,
    norm_fn = f'{DATA_DIR}{NORMFILE}',
    input_transform = ('mean', 'maxrs'),
    output_transform = scale_dict,
    batch_size=1024,
    shuffle=True,
    normalize_flag=True,
    lev=lev,
    hyam=hyam,hybm=hybm,
    rh_trans = True,t2tns_trans=False,
    lhflx_trans=False,
    scaling=False,
    interpolate=False,
    inp_subRH=train_gen_RH.input_transform.sub,inp_divRH=train_gen_RH.input_transform.div
)

valid_gen[0][0].shape
model = Sequential()
model.add(Input(shape=(76,)))
model.add(Dense(320, activation='relu'))
for i in range(6):
    model.add(Dense(320, activation='relu'))
model.add(Dense(52, activation='linear'))
opt = tf.keras.optimizers.Adam()
model.compile(optimizer=opt, loss='mse')
# model.compile(tf.keras.optimizers.Adam(), loss="mse")
path_HDF5 = '/DFS-L/DATA/pritchard/ankitesg/models/'
earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
mcp_save = ModelCheckpoint(path_HDF5+'RH_RG_Model1_V1.h5',save_best_only=True, monitor='val_loss', mode='min')
with tf.device('/gpu:1'):
    Nep = 12
    model.fit_generator(train_gen, epochs=Nep, validation_data=valid_gen
                        ,callbacks=[earlyStopping, mcp_save])<jupyter_output>Epoch 1/12
42117/42120 [============================>.] - ETA: 0s - batch: 21058.0000 - size: 1024.0000 - loss: 289.9785WARNING:tensorflow:From /export/home/ankitesg/anaconda3/envs/CBrainEnv2/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_v1.py:2048: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
42120/42120 [==============================] - 1945s 46ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 289.9762 - val_loss: 282.8817
Epoch 2/12
42120/42120 [==============================] - 1858s 44ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 259.9124 - val_loss: 259.0720
Epoch 3/12
42120/42120 [==============================] - 1968s 47ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 252.4986 - val_loss: 255.5822
Epoch 4/12
42120/42120 [==========================[...]<jupyter_text>Model 4<jupyter_code>in_vars_RH = ['RH','TBP','CLDLIQBP','CLDICEBP','PS', 'SOLIN', 'SHFLX', 'LHFLX']
BASE_DIR = '/DFS-L/DATA/pritchard/ankitesg/'
in_vars = ['QBP','TBP','CLDLIQBP','CLDICEBP','PS', 'SOLIN', 'SHFLX', 'LHFLX']
out_vars = ['NN2L_FLWDS', 'NN2L_PRECC', 
            'NN2L_PRECSC', 'NN2L_SOLL', 'NN2L_SOLLD', 'NN2L_SOLS', 'NN2L_SOLSD', 'NN2L_NETSW']
TRAINFILE = 'RG_SP_M4K_train_shuffle.nc'
NORMFILE = 'RG_SP_M4K_NORM_norm.nc'
VALIDFILE = 'RG_SP_M4K_valid.nc'
train_gen = DataGeneratorClimInvRealGeo(
    data_fn = f'{DATA_DIR}{TRAINFILE}',
    input_vars = in_vars,
    output_vars = out_vars,
    norm_fn = f'{DATA_DIR}{NORMFILE}',
    input_transform = ('mean', 'maxrs'),
    output_transform = scale_dict,
    batch_size=1024,
    shuffle=True,
    normalize_flag=True,
    lev=lev,
    hyam=hyam,hybm=hybm,
    rh_trans = True,t2tns_trans=False,
    lhflx_trans=False,
    scaling=False,
    interpolate=False,
    inp_subRH=train_gen_RH.input_transform.sub,inp_divRH=train_gen_RH.input_transform.div
)

valid_gen = DataGeneratorClimInvRealGeo(
    data_fn = f'{DATA_DIR}{VALIDFILE}',
    input_vars = in_vars,
    output_vars = out_vars,
    norm_fn = f'{DATA_DIR}{NORMFILE}',
    input_transform = ('mean', 'maxrs'),
    output_transform = scale_dict,
    batch_size=1024,
    shuffle=True,
    normalize_flag=True,
    lev=lev,
    hyam=hyam,hybm=hybm,
    rh_trans = True,t2tns_trans=False,
    lhflx_trans=False,
    scaling=False,
    interpolate=False,
    inp_subRH=train_gen_RH.input_transform.sub,inp_divRH=train_gen_RH.input_transform.div
)

valid_gen[0][1].shape
model = Sequential()
model.add(Input(shape=(76,)))
model.add(Dense(320, activation='relu'))
for i in range(4):
    model.add(Dense(320, activation='relu'))
model.add(Dense(8, activation='linear'))
opt = tf.keras.optimizers.Adam()
model.compile(optimizer=opt, loss='mse')
# model.compile(tf.keras.optimizers.Adam(), loss="mse")
path_HDF5 = '/DFS-L/DATA/pritchard/ankitesg/models/'
earlyStopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
mcp_save = ModelCheckpoint(path_HDF5+'RH_RG_Model4_V1.h5',save_best_only=True, monitor='val_loss', mode='min')
with tf.device('/gpu:1'):
    Nep = 10
    model.fit_generator(train_gen, epochs=Nep, validation_data=valid_gen
                        ,callbacks=[earlyStopping, mcp_save])<jupyter_output>Epoch 1/10
42120/42120 [==============================] - 783s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 835.5417 - val_loss: 738.5516
Epoch 2/10
42120/42120 [==============================] - 786s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 671.7713 - val_loss: 688.3112
Epoch 3/10
42120/42120 [==============================] - 788s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 644.4573 - val_loss: 673.6551
Epoch 4/10
42120/42120 [==============================] - 790s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 626.5872 - val_loss: 645.4369
Epoch 5/10
42120/42120 [==============================] - 814s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 615.4309 - val_loss: 705.3048
Epoch 6/10
42120/42120 [==============================] - 798s 19ms/step - batch: 21059.5000 - size: 1024.0000 - loss: 605.9858 - val_loss: 651.3911
Epoch 7/10
42120/42120 [==============================] - 897s 21ms/step - batch: 21059.5000 - size: 1024.[...]
{'fraction_non_alphanumeric': 0.19132012280212113, 'fraction_numerical': 0.08791515489813005, 'mean_word_length': 5.0125838926174495, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '77257', 'license': 'MIT', 'n_tokens_mistral': 3293, 'n_tokens_neox': 2789, 'n_words': 426, 'path': 'notebooks/ankitesh-devlog/23_Four_models.ipynb', 'repo_name': 'ankitesh97/CBRAIN-CAM', 'size': '7166'}