<jupyter_start><jupyter_text>Permutation Feature Importance<jupyter_code>file = open('../models/RFC.save', 'rb')
grid, X_test, y_test = pickle.load(file)
file.close()

np.random.seed(42)
ftr_names = X_test.columns
nr_runs = 10
scores = np.zeros([len(ftr_names),nr_runs])

test_score = grid.score(X_test,y_test)
baseline_score = f1_score(y_test, np.zeros(len(y_test)), average='weighted')
print('test score = ',test_score)
print('test baseline = ', baseline_score)
# loop through the features
for i in range(len(ftr_names)):
    print('shuffling '+str(ftr_names[i]))
    acc_scores = []
    for j in range(nr_runs):
        X_test_shuffled = X_test.copy()
        X_test_shuffled[ftr_names[i]] = np.random.permutation(X_test[ftr_names[i]].values)
        acc_scores.append(grid.score(X_test_shuffled,y_test))
    print('   shuffled test score:',np.around(np.mean(acc_scores),3),'+/-',np.around(np.std(acc_scores),3))
    scores[i] = acc_scores
import matplotlib.pyplot as plt
sorted_indcs = np.argsort(np.mean(scores,axis=1))
top_ten = sorted_indcs[:10]
feature_names = df.columns
xs = [scores[i] for i in top_ten]
ys = [ftr_names[i] for i in top_ten]
plt.rcParams.update({'font.size': 14})
plt.figure(figsize=(8,6))
plt.boxplot(xs,labels=ys,vert=False)
plt.axvline(test_score,label='test score')
plt.title("Permutation Importances (test set)")
plt.xlabel('score with perturbed feature')
plt.xticks(rotation=45)
plt.legend()
txt = "Fig.5 10 most important features in the Random Forest Classifier"
plt.figtext(0.5, -0.05, txt, wrap=True, horizontalalignment='center')
plt.tight_layout()
plt.savefig("../figures/permutation_feature_importance.png")
plt.show()<jupyter_output><empty_output><jupyter_text>Shap Value<jupyter_code>X_test_transformed = grid.best_estimator_[0].transform(X_test)
print(X.shape)
print(X_test.shape)
print(X_test_transformed.shape)
shap.initjs() # required for visualizations later on
# create the explainer object with the random forest model
explainer = shap.TreeExplainer(grid.best_estimator_[1])
# transform the test set
X_test_transformed = grid.best_estimator_[0].transform(X_test)
print(np.shape(X_test_transformed))
# calculate shap values on the first 1000 points in the test
shap_values = explainer.shap_values(X_test_transformed[:1000])
print(np.shape(shap_values))
feature_names = X_test.columns.values
len(feature_names)
shap.summary_plot(shap_values, X_test_transformed[:1000],feature_names = feature_names, show=False)
# plt.savefig("../figures/summary_plot.png", bbox_inches = 'tight')<jupyter_output><empty_output><jupyter_text>point shap value<jupyter_code>import matplotlib.pyplot as plt
point = 1
temp = pd.DataFrame(data=X_test_transformed,columns = X_test.columns)
shap.force_plot(explainer.expected_value[point], shap_values[point], temp.iloc[point], show=False)
# plt.savefig("../figures/force_plot.png", bbox_inches = 'tight')
print(X_test.iloc[point])
print(ordinal_owners[0][int(y_test[point])])<jupyter_output>required_age                         0
achievements                        20
positive_ratings                  5139
negative_ratings                   350
average_playtime                   245
median_playtime                    200
price                             0.79
total_ratings                     5489
ratings_ratio                 0.936236
weighted_rating                90.1191
release_year                      2015
price_categories            very cheap
multi_player                         1
single_player                        1
action                               1
adventure                            0
casual                               1
indie                                1
massively_multiplayer                0
rpg                                  0
racing                               0
simulation                           0
sports                               0
strategy                             0
choice_of_games                      0
dexion_games             [...]<jupyter_text>Curves<jupyter_code>from sklearn.preprocessing import label_binarize
y_score = grid.predict_proba(X_test)
y_binary = label_binarize(y_test, classes=[*range(n_classes)])
precision = dict()
recall = dict()
n_classes = len(df['owners'].value_counts())
plt.figure(figsize=(12,8))
for i in range(n_classes):
    precision[i], recall[i], _ = precision_recall_curve(y_binary[:, i],
                                                        y_score[:, i])
    plt.plot(recall[i], precision[i], lw=2, label='class {}'.format(i))
    
plt.xlabel("recall")
plt.ylabel("precision")
plt.legend(loc="best")
plt.title("precision vs. recall curve")
txt = 'Fig.6 The result graph based on setting each class as a binary classification in the method of "one against all"'
plt.figtext(0.5, -0.01, txt, wrap=True, horizontalalignment='center')
plt.savefig("../figures/precision-recall-curve.png", bbox_inches = 'tight')
plt.show()
fpr = dict()
tpr = dict()

for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_binary[:, i],
                                  y_score[:, i])
    plt.plot(fpr[i], tpr[i], lw=2, label='class {}'.format(i))

plt.xlabel("false positive rate")
plt.ylabel("true positive rate")
plt.legend(loc="best")
plt.title("ROC curve")
plt.show()
y_pred = grid.predict(X_test)
multilabel_confusion_matrix(y_test, y_pred)
fig, ax = plt.subplots(figsize=(12,12))
plot_confusion_matrix(grid,X_test,y_test, ax = ax)
txt = 'Fig.7 The confusion matrix of 9 classes in the target variable.'
plt.figtext(0.5, 0.1, txt, wrap=True, horizontalalignment='center')
plt.savefig("../figures/confusion_matrix.png", bbox_inches = 'tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12371317003904864, 'fraction_numerical': 0.020411785587504436, 'mean_word_length': 3.308103975535168, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '366511', 'license': 'MIT', 'n_tokens_mistral': 1999, 'n_tokens_neox': 1860, 'n_words': 362, 'path': 'src/featre_importance.ipynb', 'repo_name': 'Enmin/steam_game_sales', 'size': '5634'}