<jupyter_start><jupyter_text>prediction on test data<jupyter_code>test_df  = pd.read_csv('../data/balanced-classes/test.csv')
test_df.shape
test_df.head()
X_test = test_df.iloc[:,0:8]
y_test =  test_df.iloc[:,8]
y_test.shape
X_test.shape
y_pred_test = model.predict(X_test)
get_metrics(y_test, y_pred_test)
display_classification_report(y_test, y_pred_test)
conf_mat_test = confusion_matrix(y_test, y_pred_test)

dataframe_test = pd.DataFrame(conf_mat_test)
dataframe_test.head()
# Create heatmap
sns.heatmap(dataframe_test, annot=True, cbar=None, cmap="Blues", fmt='g');
plt.title("Confusion Matrix"), plt.tight_layout();
plt.ylabel("True Class"), plt.xlabel("Predicted Class");
plt.show();<jupyter_output><empty_output><jupyter_text>prediction on full data set<jupyter_code>full_df  = pd.read_csv('../data/data-with-features.csv')
full_df.shape
full_df.head()
X_full_df = full_train_df.iloc[:,0:8]
y_full_df =  full_train_df.iloc[:,8]
X_full_df.shape
y_full_df.shape
y_pred_full_df = model.predict(X_full_df)
get_metrics(y_full_df, y_pred_full_df)
display_classification_report(y_full_df, y_pred_full_df)
conf_mat_full = confusion_matrix(y_full_df, y_pred_full_df)

dataframe_full = pd.DataFrame(conf_mat_full)
dataframe_full.head()
# Create heatmap
sns.heatmap(dataframe_full, annot=True, cbar=None, cmap="Blues", fmt='g');
plt.title("Confusion Matrix"), plt.tight_layout();
plt.ylabel("True Class"), plt.xlabel("Predicted Class");
plt.show();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14227642276422764, 'fraction_numerical': 0.0040650406504065045, 'mean_word_length': 9.116438356164384, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '514429', 'license': 'MIT', 'n_tokens_mistral': 609, 'n_tokens_neox': 601, 'n_words': 80, 'path': 'team-i-catch/deepak/notebooks/random-forest.ipynb', 'repo_name': 'DeepakRautelaTR/amazon-sagemaker-architecting-for-ml', 'size': '1476'}