<jupyter_start><jupyter_text>Functions<jupyter_code>def create_dataset(dataset, look_back=1, response_variable_index=0, multi_feature = True):
  dataX, dataY = [], []
  if multi_feature :
    for i in range(len(dataset)-look_back-1):
      a = dataset[i:(i+look_back)]
      dataX.append(a)
      dataY.append(dataset[i + look_back, response_variable_index])
  else:
    for i in range(len(dataset)-look_back-1):
      a = dataset[i:(i+look_back-1),response_variable_index]
      dataX.append(a)
      dataY.append(dataset[i + look_back, response_variable_index])
  return np.array(dataX), np.array(dataY)
def prepare(df, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8):
  df = df.astype("float32")
  scaler = MinMaxScaler(feature_range=scaling_range)
  dataset = scaler.fit_transform(df)
  X, Y = create_dataset(dataset, time_step, response_variable_index, multi_feature = True)
  # split into train and test sets
  train_size = int(len(dataset) * data_split_ratio)
  test_size = len(dataset) - train_size
  trainX, testX = X[0:train_size,:], X[train_size:len(dataset),:]
  trainY, testY = Y[0:train_size], Y[train_size:len(dataset)]
  #print(trainX.shape)
  #print(testX.shape)
  # reshape input to be [samples, time steps, features]
  if not multi_feature:
    trainX = np.reshape(trainX, (trainX.shape[0],trainX.shape[1],1))
    testX = np.reshape(testX, (testX.shape[0], testX.shape[1],1))
    #print(trainX.shape)
  X_train=trainX
  X_test=testX
  y_train=trainY.reshape(-1,1)
  y_test=testY.reshape(-1,1)
  return X_train,X_test, y_train, y_test
def model_cnn():
  model_cnn = keras.Sequential()
  model_cnn.add(Conv1D(filters=16,
        kernel_size=3,
        input_shape=(X_train.shape[1], X_train.shape[2])
            )
                  )
  model_cnn.add(LeakyReLU(alpha=0.1))
  model_cnn.add(MaxPooling1D(pool_size=1))
  model_cnn.add(Dropout(rate=0.2))
  model_cnn.add(Flatten())
  model_cnn.add(Dense(1))
  model_cnn.compile(loss='mae', optimizer='adam')
  return model_cnn

def fit(model, X_train, X_test, y_train, y_test):
  history = model.fit(X_train, y_train, epochs=100, batch_size=16, validation_data=(X_test, y_test), callbacks=[EarlyStopping(monitor= "val_loss", patience=10)], shuffle=False)
  return history

def predict(model, y_test):
  y_test_pre = model.predict(X_test)
  return y_test_pre
def visualize(y_test, y_pre, title):
  results = pd.DataFrame()
  #results['Date']= date_test.reset_index().Date.iloc[1:-1]
  #results["Date"] = date_test
  results["predicted"]=[i[0] for i in y_pre][1:] 
  results["true"] = y_test[:-1]
  #results = results.set_index(["Date"])
  #results.head()
  pyplot.plot(results["predicted"], label='predicted', color='red')
  pyplot.plot(results["true"], label='true', color= 'green')
  pyplot.legend()
  pyplot.title(title)
  pyplot.show()
def evaluate(y_test, y_pre):
  true = y_test[:-1]
  predicted = [i[0] for i in y_pre][1:]
  rmse = np.sqrt(mean_squared_error(true, predicted))
  mae = mean_absolute_error(true, predicted)
  #print(len(true))
  #print(len(predicted))
  return rmse, mae<jupyter_output><empty_output><jupyter_text>Keras Tuner<jupyter_code>def build_model(hp):  
  model = keras.Sequential([
    keras.layers.Conv1D(
        hp.Int("input_units", min_value=16, max_value=256, step=16),
        kernel_size= hp.Choice('conv_1_kernel', values = [1,5]),
        input_shape= (X_train.shape[1], X_train.shape[2])
    ),
    keras.layers.LeakyReLU(alpha=0.1),
    keras.layers.MaxPooling1D(1),
   
    keras.layers.Flatten(),
    keras.layers.Dropout(0.2)
    
  ])
  
  model.compile(optimizer=keras.optimizers.Adam(hp.Choice('learning_rate', values=[1e-2, 1e-3])),
              loss='mae',
              metrics=['mae'])
  print(model.summary())
  return model
log_dir = "logs"<jupyter_output><empty_output><jupyter_text>Import Data<jupyter_code># Maharashtra
df_Ma = pd.read_csv("https://raw.githubusercontent.com/sureshkuc/Data-Science-in-Life-Science-Project/main/Indian-States-Covid19-Datasets/Maharashtra.csv", parse_dates=["Date"]).drop(columns =["Unnamed: 0"])
# Karnataka
df_Ka = pd.read_csv("https://raw.githubusercontent.com/sureshkuc/Data-Science-in-Life-Science-Project/main/Indian-States-Covid19-Datasets/Karnataka.csv", parse_dates=["Date"]).drop(columns =["Unnamed: 0"])
# Kerala
df_Ke = pd.read_csv("https://raw.githubusercontent.com/sureshkuc/Data-Science-in-Life-Science-Project/main/Indian-States-Covid19-Datasets/Kerala.csv", parse_dates=["Date"]).drop(columns =["Unnamed: 0"])
# Tamil Nadu
df_TN = pd.read_csv("https://raw.githubusercontent.com/sureshkuc/Data-Science-in-Life-Science-Project/main/Indian-States-Covid19-Datasets/Tamil-Nadu.csv", parse_dates=["Date"]).drop(columns =["Unnamed: 0"])
# Uttar Pradesh
df_UP = pd.read_csv("https://raw.githubusercontent.com/sureshkuc/Data-Science-in-Life-Science-Project/main/Indian-States-Covid19-Datasets/Uttar-Pradesh.csv", parse_dates=["Date"]).drop(columns =["Unnamed: 0"])


df_Ma = df_Ma[["Date", "Confirmed", "Recovered", "Deceased", "New_Confirmerd", "New_Deaths", "New_Recovered"]]
df_Ka = df_Ka[["Date", "Confirmed", "Recovered", "Deceased", "New_Confirmerd", "New_Deaths", "New_Recovered"]]
df_Ke = df_Ke[["Date", "Confirmed", "Recovered", "Deceased", "New_Confirmerd", "New_Deaths", "New_Recovered"]]
df_TN = df_TN[["Date", "Confirmed", "Recovered", "Deceased", "New_Confirmerd", "New_Deaths", "New_Recovered"]]
df_UP = df_UP[["Date", "Confirmed", "Recovered", "Deceased", "New_Confirmerd", "New_Deaths", "New_Recovered"]]
df_Ma = df_Ma[df_Ma["Date"] > "2020-03-09"]
df_Ka = df_Ka[df_Ka["Date"] > "2020-03-09"]
df_Ke = df_Ke[df_Ke["Date"] > "2020-03-09"]
df_TN = df_TN[df_TN["Date"] > "2020-03-09"]
df_UP = df_UP[df_UP["Date"] > "2020-03-09"]
df_Ma = df_Ma.set_index("Date")
df_Ka = df_Ka.set_index("Date")
df_Ke = df_Ke.set_index("Date")
df_TN = df_TN.set_index("Date")
df_UP = df_UP.set_index("Date")

RMSE_cnn = []
MAE_cnn = []
RMSE_tun = []
MAE_tun = []
states = ["Maharashtra", "Karnataka", "Kerala", "Tamil Nadu", "Uttar Pradesh"]<jupyter_output><empty_output><jupyter_text>Maharashtra<jupyter_code>X_train,X_test, y_train, y_test = prepare(df_Ma, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8)
print("X_train.shape", X_train.shape)
print("X_test.shape", X_test.shape)
print("y_train.shape", y_train.shape)
print("y_test.shape", y_test.shape)
model_CNN = model_cnn()
fit(model_CNN, X_train, X_test, y_train, y_test)
model_CNN.summary()
y_test_pre_cnn = predict(model_CNN, y_test)
print("CNN RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn)
RMSE_cnn.append(rmse)
MAE_cnn.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn, "results of CNN")

tuner =  RandomSearch( build_model, objective= "mae", max_trials=2, executions_per_trial= 2, directory= "log_dir", overwrite=True)
tuner.search(X_train, y_train, epochs= 100, batch_size= 16, validation_data=(X_test, y_test))
tuner.results_summary()
model_cnn_tuner = keras.Sequential()
model_cnn_tuner.add(Conv1D(filters=144,
  kernel_size=5,
  input_shape=(X_train.shape[1], X_train.shape[2])
              )
                    )
model_cnn_tuner.add(LeakyReLU(alpha=0.1))
model_cnn_tuner.add(MaxPooling1D(pool_size=1))
model_cnn_tuner.add(Dropout(rate=0.2))
model_cnn_tuner.add(Flatten())
model_cnn_tuner.add(Dense(1))
model_cnn_tuner.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
              loss="mae",
              metrics=["mae"])
fit(model_cnn_tuner, X_train, X_test, y_train, y_test)
y_test_pre_cnn_t = predict(model_cnn_tuner, y_test)
print("CNN_tun RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn_t)
RMSE_tun.append(rmse)
MAE_tun.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn_t, "results of CNN_tun")<jupyter_output>CNN_tun RESULTS
RMSE =  0.0869774
MAE =  0.08031924<jupyter_text>Karnataka<jupyter_code>X_train,X_test, y_train, y_test = prepare(df_Ka, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8)
model_CNN = model_cnn()
fit(model_CNN, X_train, X_test, y_train, y_test)
y_test_pre_cnn = predict(model_CNN, y_test)
print("CNN RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn)
RMSE_cnn.append(rmse)
MAE_cnn.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn, "results of CNN")

log_dir = "logs"
tuner =  RandomSearch( build_model, objective= "mae", max_trials=2, executions_per_trial= 2, directory= "log_dir", overwrite=True)
tuner.search(X_train, y_train, epochs=100, batch_size= 16, validation_data=(X_test, y_test))
tuner.results_summary()
model_cnn_tuner = keras.Sequential()
model_cnn_tuner.add(Conv1D(filters=96,
  kernel_size=5,
  input_shape=(X_train.shape[1], X_train.shape[2])
              )
                    )
model_cnn_tuner.add(LeakyReLU(alpha=0.1))
model_cnn_tuner.add(MaxPooling1D(pool_size=1))
model_cnn_tuner.add(Dropout(rate=0.2))
model_cnn_tuner.add(Flatten())
model_cnn_tuner.add(Dense(1))
model_cnn_tuner.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
              loss="mae",
              metrics=["mae"])
fit(model_cnn_tuner, X_train, X_test, y_train, y_test)
y_test_pre_cnn_t = predict(model_cnn_tuner, y_test)
print("CNN_tun RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn_t)
RMSE_tun.append(rmse)
MAE_tun.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn_t, "results of CNN_tun")<jupyter_output>CNN_tun RESULTS
RMSE =  0.081083015
MAE =  0.06909799<jupyter_text>Kerala<jupyter_code>X_train,X_test, y_train, y_test = prepare(df_Ke, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8)
model_CNN = model_cnn()
fit(model_CNN, X_train, X_test, y_train, y_test)
y_test_pre_cnn = predict(model_CNN, y_test)
print("CNN RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn)
RMSE_cnn.append(rmse)
MAE_cnn.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn, "results of CNN")

tuner =  RandomSearch( build_model, objective= "mae", max_trials=2, executions_per_trial= 2, directory= "log_dir", overwrite=True)
tuner.search(X_train, y_train, epochs=100, batch_size=16, validation_data=(X_test, y_test))
tuner.results_summary()
model_cnn_tuner = keras.Sequential()
model_cnn_tuner.add(Conv1D(filters=192,
  kernel_size=5,
  input_shape=(X_train.shape[1], X_train.shape[2])
              )
                    )
model_cnn_tuner.add(LeakyReLU(alpha=0.1))
model_cnn_tuner.add(MaxPooling1D(pool_size=1))
model_cnn_tuner.add(Dropout(rate=0.2))
model_cnn_tuner.add(Flatten())
model_cnn_tuner.add(Dense(1))
model_cnn_tuner.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
              loss="mae",
              metrics=["mae"])
fit(model_cnn_tuner, X_train, X_test, y_train, y_test)
y_test_pre_cnn_t = predict(model_cnn_tuner, y_test)
print("CNN_tun RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn_t)
RMSE_tun.append(rmse)
MAE_tun.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn_t, "results of CNN_tun")<jupyter_output>CNN_tun RESULTS
RMSE =  0.05059366
MAE =  0.044362705<jupyter_text><NAME><jupyter_code>X_train,X_test, y_train, y_test = prepare(df_TN, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8)
model_CNN = model_cnn()
fit(model_CNN, X_train, X_test, y_train, y_test)
y_test_pre_cnn = predict(model_CNN, y_test)
print("CNN RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn)
RMSE_cnn.append(rmse)
MAE_cnn.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn, "results of CNN")

tuner =  RandomSearch( build_model, objective= "mae", max_trials=2, executions_per_trial= 2, directory= "log_dir", overwrite=True)
tuner.search(X_train, y_train, epochs=100, batch_size=16, validation_data=(X_test, y_test))
tuner.results_summary()
model_cnn_tuner = keras.Sequential()
model_cnn_tuner.add(Conv1D(filters=32,
  kernel_size=5,
  input_shape=(X_train.shape[1], X_train.shape[2])
              )
                    )
model_cnn_tuner.add(LeakyReLU(alpha=0.1))
model_cnn_tuner.add(MaxPooling1D(pool_size=1))
model_cnn_tuner.add(Dropout(rate=0.2))
model_cnn_tuner.add(Flatten())
model_cnn_tuner.add(Dense(1))
model_cnn_tuner.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
              loss="mae",
              metrics=["mae"])
fit(model_cnn_tuner, X_train, X_test, y_train, y_test)
y_test_pre_cnn_t = predict(model_cnn_tuner, y_test)
print("CNN_tun RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn_t)
RMSE_tun.append(rmse)
MAE_tun.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn_t, "results of CNN_tun")<jupyter_output>CNN_tun RESULTS
RMSE =  0.017732266
MAE =  0.015831325<jupyter_text><NAME><jupyter_code>X_train,X_test, y_train, y_test = prepare(df_UP, scaling_range=(0,1),time_step=5,multi_feature=True, response_variable_index=0,data_split_ratio=0.8)
model_CNN = model_cnn()
fit(model_CNN, X_train, X_test, y_train, y_test)
y_test_pre_cnn = predict(model_CNN, y_test)
print("CNN RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn)
RMSE_cnn.append(rmse)
MAE_cnn.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn, "results of CNN")

tuner =  RandomSearch( build_model, objective= "mae", max_trials=2, executions_per_trial= 2, directory= "log_dir", overwrite=True)
tuner.search(X_train, y_train, epochs=100, batch_size=16, validation_data=(X_test, y_test))
tuner.results_summary()
model_cnn_tuner = keras.Sequential()
model_cnn_tuner.add(Conv1D(filters=112,
  kernel_size=5,
  input_shape=(X_train.shape[1], X_train.shape[2])
              )
                    )
model_cnn_tuner.add(LeakyReLU(alpha=0.1))
model_cnn_tuner.add(MaxPooling1D(pool_size=1))
model_cnn_tuner.add(Dropout(rate=0.2))
model_cnn_tuner.add(Flatten())
model_cnn_tuner.add(Dense(1))
model_cnn_tuner.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001),
              loss="mae",
              metrics=["mae"])
fit(model_cnn_tuner, X_train, X_test, y_train, y_test)
y_test_pre_cnn_t = predict(model_cnn_tuner, y_test)
print("CNN_tun RESULTS")
rmse, mae = evaluate(y_test, y_test_pre_cnn_t)
RMSE_tun.append(rmse)
MAE_tun.append(mae)
print("RMSE = ", rmse)
print("MAE = ", mae)

visualize(y_test, y_test_pre_cnn_t, "results of CNN_tun")<jupyter_output>CNN_tun RESULTS
RMSE =  0.09005914
MAE =  0.06727739<jupyter_text>Comparison<jupyter_code>compare = pd.DataFrame()
compare["State"] = states
compare["rmse_cnn"] = RMSE_cnn
compare["rmse_cnn_t"] = RMSE_tun
compare["mae_cnn"] = MAE_cnn
compare["mae_cnn_t"] = MAE_tun
compare = compare.round(4)
MAE_cnn = [0.0765, 0.1056, 0.1014, 0.1084, 0.0552]
RMSE_cnn = [0.0864, 0.1543, 0.1347, 0.1418, 0.0738]
compare.head()
compare.mean(axis=0)
compare.to_latex()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15304894356779888, 'fraction_numerical': 0.028684140144423644, 'mean_word_length': 6.278345498783455, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 39, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '71455', 'license': 'MIT', 'n_tokens_mistral': 6712, 'n_tokens_neox': 6360, 'n_words': 847, 'path': 'Implementation/Keras/CNN_Tuner.ipynb', 'repo_name': 'sureshkuc/Data-Science-in-Life-Science-Project', 'size': '14967'}