<jupyter_start><jupyter_text>Preparing Training Dataset<jupyter_code>pm1_output_dataset = pd.read_csv("output/" + PM1_FILENAME + EXT, skipinitialspace=True)
pm2_output_dataset = pd.read_csv("output/" + PM2_FILENAME + EXT, skipinitialspace=True)

print("Predicted Column of pm1 : " + str(pm1_output_dataset.columns[0:pm1_output_dataset.shape[1]][2]))
print("Predicted Column of pm2 : " + str(pm2_output_dataset.columns[0:pm2_output_dataset.shape[1]][1]))

pm1_output_dataset.Predicted
d = {'PM1-Output': pm1_output_dataset.Predicted, 'PM2-Output': pm2_output_dataset.Predicted}
df = pd.DataFrame(data=d)
df.head()

training_dataset = np.array(df)
# To-be Predicted variable
Y = pm1_output_dataset.Actual
Y = Y.round(5)

# Other data
X = training_dataset
#X = dataset
#X.statusS = X.statusS.round(5)

# Splitting data
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.67, shuffle=False)
X_train = np.array(X_train)
X_test = np.array(X_test)
Y_train = np.array(Y_train)
Y_test = np.array(Y_test)

# Data scaling
min_max_scaler = MinMaxScaler()
X_train = min_max_scaler.fit_transform(X_train)
# Training
regressor = SupervisedDBNRegression(hidden_layers_structure=[3],
                                    learning_rate_rbm=0.01,
                                    learning_rate=0.01,
                                    n_epochs_rbm=3,
                                    n_iter_backprop=5,
                                    batch_size=16,
                                    activation_function='relu')
regressor.fit(X_train, Y_train)

# Test
X_test = min_max_scaler.transform(X_test)
Y_pred = regressor.predict(X_test)
print('Done.\nR-squared: %f\nMSE: %f' % (r2_score(Y_test, Y_pred), mean_squared_error(Y_test, Y_pred)))

print(len(Y_pred))
temp = []
for i in range(len(Y_pred)):
    temp.append(Y_pred[i][0])
d = {'Predicted': temp, 'Actual': Y_test}

df = pd.DataFrame(data=d)
df.head()
df
# Save the model
regressor.save('models/fusion-center.pkl')

# # Restore
# classifier = SupervisedDBNClassification.load('model.pkl')

# # Test
# Y_pred = classifier.predict(X_test)
# print('Done.\nAccuracy: %f' % accuracy_score(Y_test, Y_pred))

FILENAME = "merged_mmda_wwo_Taft Ave._2015"
EXT = ".csv"
df.to_csv("output/fusion-center_output" + FILENAME + EXT, encoding='utf-8', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13136499786050493, 'fraction_numerical': 0.019255455712451863, 'mean_word_length': 3.8912133891213387, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '577644', 'license': 'MIT', 'n_tokens_mistral': 903, 'n_tokens_neox': 866, 'n_words': 135, 'path': 'FUSION-CENTER.ipynb', 'repo_name': 'fuouo/thesis-models', 'size': '2337'}