<jupyter_start><jupyter_text>Train<jupyter_code>gbm = GradientBoostingClassifier(random_state=0)
gbm.fit(X_trn, Y_trn)<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>plt.figure(figsize=(12, 8))
feat_imp = pd.Series(gbm.feature_importances_, X_trn.columns).sort_values(ascending=False)
feat_imp.plot(kind='bar', title='Feature Importances')
plt.ylabel('Feature Importance Score')
Y_dev.value_counts() / len(Y_dev)
Y_pred = gbm.predict(X_dev)
plt.figure(figsize=(15, 7))

ax1 = plt.subplot(1, 2, 1)

value_counts = Counter(Y_dev)
_ = ax1.pie(x = list(value_counts.values()), 
            labels = list(value_counts.keys()),
            autopct='%1.1f%%', 
            labeldistance = 1.1)
ax1.set_title('Development Set Label')

ax2 = plt.subplot(1, 2, 2)

value_counts = Counter(Y_pred)
_ = ax2.pie(x = list(value_counts.values()), 
            labels = list(value_counts.keys()),
            autopct='%1.1f%%',
            labeldistance = 1.1)

ax2.set_title('Predicted Label')

plt.show()
print("Accuracy : %.4g" % metrics.accuracy_score(Y_dev.values, Y_pred))
df_m = pd.DataFrame(confusion_matrix(Y_dev, Y_pred), columns=["predicted_%i"%(i + 1) for i in range(10)])
df_m.index = df_m.index.map(lambda x : "true_" + str(x + 1))

df_m
np.diag(np.matrix(df_m.values))
# recall
np.diag(np.matrix(df_m.values)) / df_m.sum(axis=1)
# precision
np.diag(np.matrix(df_m.values)) / df_m.sum(axis=0)
plt.matshow(confusion_matrix(Y_dev, Y_pred), cmap=plt.cm.gray)
plt.savefig("figs/confusion_heatmap.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16796368352788585, 'fraction_numerical': 0.02204928664072633, 'mean_word_length': 5.221774193548387, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '563640', 'license': 'MIT', 'n_tokens_mistral': 667, 'n_tokens_neox': 634, 'n_words': 83, 'path': 'case_study/shutterfly/gbm_benchmark_2.ipynb', 'repo_name': 'shawlu95/Data_Science_Toolbox', 'size': '1542'}