<jupyter_start><jupyter_text>all variables, including categorical<jupyter_code>df = df[['Pclass','Sex','SibSp','Embarked','Age','Fare','Survived']]

df = pd.concat([df,pd.get_dummies(df['Pclass'], prefix='Pclass',dummy_na=True)],axis=1).drop(['Pclass'],axis=1)
df = pd.concat([df,pd.get_dummies(df['Sex'], prefix='Sex',dummy_na=True)],axis=1).drop(['Sex'],axis=1)
df = pd.concat([df,pd.get_dummies(df['SibSp'], prefix='SibSp',dummy_na=True)],axis=1).drop(['SibSp'],axis=1)
df = pd.concat([df,pd.get_dummies(df['Embarked'], prefix='Embarked',dummy_na=True)],axis=1).drop(['Embarked'],axis=1)
df.head()
np.random.seed(42)

data=df.drop(['Survived'],axis=1)
target = df['Survived']

X_train, X_test, y_train, y_test = train_test_split(data.values, target.values, test_size=0.4)
clf = XGBClassifier()
clf.fit(X_train,y_train)

y_preds = clf.predict_proba(X_test)
preds = y_preds[:,1]

# fpr means false-positive-rate
# tpr means true-positive-rate
fpr, tpr, _ = metrics.roc_curve(y_test, preds)

auc_score = metrics.auc(fpr, tpr)

plt.title('ROC Curve')
plt.plot(fpr, tpr, label='AUC = {:.3f}'.format(auc_score))

# it's helpful to add a diagonal to indicate where chance 
# scores lie (i.e. just flipping a coin)
plt.plot([0,1],[0,1],'r--')

plt.xlim([-0.1,1.1])
plt.ylim([-0.1,1.1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')

plt.legend(loc='lower right')
plt.show()
importance_data = sorted(list(zip(data.columns,clf.feature_importances_)),key=lambda tpl:tpl[1],reverse=True)

xs = range(len(importance_data))
labels = [x for (x,_) in importance_data]

ys = [y for (_,y) in importance_data]
plt.clf()
plt.bar(xs,ys,width=0.5)
plt.xticks(xs,labels,rotation=45,ha='right')

plt.gca().grid(True)
# select both y axis and x axis
gridlines = plt.gca().get_xgridlines() + plt.gca().get_ygridlines()
# choose line width
line_width = 0.7

plt.ylabel('relative feature importance')

for line in gridlines:
    line.set_linestyle(':')
    line.set_linewidth(line_width)
plt.show()
categorical_columns_names = ['Pclass','Sex','SibSp','Embarked']

for column_name in categorical_columns_names:
    
    all_values_sum = 0
    
    for key,value in list(importances_dict.items():
        if key.startswith(column_name):
            all_values_sum += importances_dict[key]
            del(importances_dict[key])
            
    importances_dict[column_name] = all_values_sum
importance_data = sorted(list(importances_dict.items()),key=lambda tpl:tpl[1],reverse=True)

xs = range(len(importance_data))
labels = [x for (x,_) in importance_data]

ys = [y for (_,y) in importance_data]
plt.clf()
plt.bar(xs,ys,width=0.5)
plt.xticks(xs,labels,rotation=45,ha='center')

plt.gca().grid(True)
# select both y axis and x axis
gridlines = plt.gca().get_xgridlines() + plt.gca().get_ygridlines()
# choose line width
line_width = 0.7

plt.ylabel('relative feature importance')

for line in gridlines:
    line.set_linestyle(':')
    line.set_linewidth(line_width)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17815517815517815, 'fraction_numerical': 0.013986013986013986, 'mean_word_length': 6.101654846335697, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '456220', 'license': 'MIT', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1178, 'n_words': 195, 'path': 'python3/notebooks/visualizing-machine-learning-models-post/plot-importances-xgb.ipynb', 'repo_name': 'tobby2002/python-sandbox', 'size': '3003'}