<jupyter_start><jupyter_text>Feature engineering<jupyter_code>df_model['Disposition'].value_counts()
df_model.loc[df_model['Disposition'].isin(["NO ACTION POSSIBLE/NECESSARY/UNABLE TO CONTACT",
 "OTHER", "-", "RESOURCES OFFERED/DECLINED", 
 "CHRONIC COMPLAINT", "DMHP/REFERRAL (DCR)",
 "CSC/CDF (CRISIS SOLUTION CENTER/CRISIS DIVERSION FACILITY)", 
 "MENTAL HEALTH AGENCY OR CASE MANAGER NOTIFIED", 
 "VOLUNTARY COMMITTAL", "MOBILE CRISIS TEAM",
 "CRISIS CLINIC"]), 'Disposition'] = "OTHER"
df_model['Disposition'].value_counts()
df_model['Exhibiting_Behavior'].value_counts()
df_model.loc[df_model['Exhibiting_Behavior'].isin(["DISORGANIZED SPEECH/COMMUNICATION",
    "OUT OF TOUCH WITH REALITY", "HOPELESS/DEPRESSED",
    "MANIC", "HALLUCINATIONS/DELUSIONS", "PARANOID",
    "DISORIENTATION/CONFUSION","UNUSUALLY FRIGHTENED/SCARED"]), 'Exhibiting_Behavior'] = "MENTAL CRISIS"
df_model.loc[df_model['Exhibiting_Behavior'].isin(["NEGLECT/SELF CARE", 
    "SUICIDE THREAT/ATTEMPT", "SUICIDE BY COP (SBC)"]), 'Exhibiting_Behavior'] = "SELF-THREATENING"
df_model.loc[df_model['Exhibiting_Behavior'].isin([
    "PHYSICALLY THREATENING/AGGRESSIVE", 
    "VERBALLY THREATENING/AGGRESSIVE"]), 'Exhibiting_Behavior'] = "THREATENING/AGGRESSIVE"
df_model.loc[df_model['Exhibiting_Behavior'].isin([
    "OTHER", "-", "BIZARRE/UNUSUAL"]), 'Exhibiting_Behavior'] = "OTHER"
df_model['Exhibiting_Behavior'].value_counts()
df_model['Subject_Race'].value_counts()
df_model.loc[df_model['Subject_Race'].isin([
    "American Indian/Alaska Native", "Nat Hawaiian/Oth Pac Islander"]), 'Subject_Race'] = "AIAN/HPI"
df_model['Subject_Race'].value_counts()
df_model['Techniques_Used'].value_counts()
df_model.loc[df_model['Techniques_Used'].isin([
    "NONE", "-"]), 'Techniques_Used'] = "OTHER"
df_model['Techniques_Used'].value_counts()
df_model['Weapons_Involved'].value_counts()
df_model.loc[df_model['Weapons_Involved'].isin([
    "-"]), 'Weapons_Involved'] = "OTHER"
df_model.loc[df_model['Weapons_Involved'].isin([
    "TASER/STUN GUN", "MACE/PEPPER SPRAY"]), 'Weapons_Involved'] = "PERSONAL DEFENSE WEAPONS"
df_model['Weapons_Involved'].value_counts()
df_model['Officer_Race'].value_counts()
df_model.loc[df_model['Officer_Race'].isin([
    "American Indian/Alaska Native", "Nat Hawaiian/Oth Pac Islander"]), 'Officer_Race'] = "AIAN/HPI"
df_model['Officer_Race'].value_counts()
# Drop ID cols
df_model.drop(["Behavioral_Crisis_Num", "CAD_Event_ID", "Officer_Serial_Num"], axis = 1, inplace=True)
df_model.head()
#Check on the % of null values
df_model.isnull().sum()*100/df_model.shape[0]
# Remove subject Age and then remove missing values
df_model.drop(["Subject_Age"], axis = 1, inplace=True)
df_model.dropna(inplace = True)
#Check on the % of null values
df_model.isnull().sum()*100/df_model.shape[0]
df_model.shape<jupyter_output><empty_output><jupyter_text>Apply One hot encoding to categorical variables<jupyter_code>y = df_model['UoF_Indicator']
X = pd.get_dummies(df_model.loc[:,df_model.columns!='UoF_Indicator'])
X.head()
X.shape
# implementing train-test-split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=66)
y_train.value_counts()<jupyter_output><empty_output><jupyter_text>EDA on numerical variables<jupyter_code>df_train = pd.concat([y_train.reset_index(drop=True), X_train.reset_index(drop=True)], axis=1)
def boxplot_eda(df_train, response_var, feature_int):
    plt.figure(figsize=(6, 6))
    ax = sns.boxplot(x=response_var, y=feature_int, data=df_train)
    ax.set_xticklabels(ax.get_xticklabels(), rotation=0, horizontalalignment="right")
    ax.set_title(response_var)
    add_stat_annotation(ax, data=df_train, x=response_var, y=feature_int, order=['YES', 'NO'],
                        box_pairs=[("YES", "NO")],
                        test='Mann-Whitney', text_format='star', loc='inside', verbose=2)
boxplot_eda(df_train, response_var = 'UoF_Indicator', feature_int="Total_Service_Time")
boxplot_eda(df_train, response_var = 'UoF_Indicator', feature_int="Latitude")
boxplot_eda(df_train, response_var = 'UoF_Indicator', feature_int="Longitude")<jupyter_output>p-value annotation legend:
ns: 5.00e-02 < p <= 1.00e+00
*: 1.00e-02 < p <= 5.00e-02
**: 1.00e-03 < p <= 1.00e-02
***: 1.00e-04 < p <= 1.00e-03
****: p <= 1.00e-04

YES v.s. NO: Mann-Whitney-Wilcoxon test two-sided with Bonferroni correction, P_val=4.161e-06 U_stat=4.624e+05<jupyter_text>EDA on categorical variables<jupyter_code>#create contingency table
data = pd.crosstab(index=df_train['UoF_Indicator'], columns=df_train['Subject_Race_Black or African American']) #,  margins=True)
data
df_train['UoF_Indicator'].value_counts()
# Chi-square
from scipy.stats import chi2_contingency
stat, p, dof, expected = chi2_contingency(data)
# interpret p-value
alpha = 0.05
print("p value is " + str(p))
if p <= alpha:
    print('Dependent (reject H0)')
else:
    print('Independent (H0 holds true)')<jupyter_output>p value is 5.884335956572277e-16
Dependent (reject H0)<jupyter_text>Class rebalance<jupyter_code>pip install imbalanced-learn
from imblearn.over_sampling import SMOTE 
X_train_bf = X_train

sm = SMOTE(random_state=42)

X_train, y_train = sm.fit_resample(X_train, y_train)

print(f'''Shape of X before SMOTE: {X_train_bf.shape}
Shape of X after SMOTE: {X_train.shape}''')

print('\nBalance of positive and negative classes (%):')
y_train.value_counts(normalize=True) * 100
y_train.value_counts()
X_train.shape<jupyter_output><empty_output><jupyter_text>Model training<jupyter_code>from sklearn.model_selection import GridSearchCV
rfc = RandomForestClassifier(random_state = 1)
n_estimators = [100, 300, 500, 800, 1200]
max_depth = [5, 8, 15, 25, 30]
min_samples_split = [2, 5, 10, 15, 100]
min_samples_leaf = [1, 2, 5, 10] 

hyperF = dict(n_estimators = n_estimators, max_depth = max_depth,  
              min_samples_split = min_samples_split, 
             min_samples_leaf = min_samples_leaf)

gridF = GridSearchCV(rfc, hyperF, cv = 3, verbose = 1, 
                      n_jobs = -1)
bestF = gridF.fit(X_train, y_train)
bestF
bestF.best_params_
# random forest model creation
rfc = RandomForestClassifier(random_state = 1, 
                             max_depth = 30,
                             n_estimators = 500, 
                             min_samples_split = 2, 
                             min_samples_leaf = 1)
rfc.fit(X_train,y_train)
# predictions
rfc_predict = rfc.predict(X_test)
# Evaluating performance
rfc_cv_score = cross_val_score(rfc, X_train, y_train, cv=10, scoring='roc_auc')
print("=== Confusion Matrix ===")
print(confusion_matrix(y_test, rfc_predict))
print('\n')
print("=== Classification Report ===")
print(classification_report(y_test, rfc_predict))
print('\n')
print("=== All AUC Scores ===")
print(rfc_cv_score)
print('\n')
print("=== Mean AUC Score ===")
print("Mean AUC Score - Random Forest: ", rfc_cv_score.mean())
rfc.classes_
# Feature importance
plt.figure(figsize=(10, 10))
sorted_idx = rfc.feature_importances_.argsort()
plt.barh(X_train.columns[sorted_idx], rfc.feature_importances_[sorted_idx])
plt.xlabel("Random Forest Feature Importance")
## Feature importance based on feature permutation
plt.figure(figsize=(10, 10))
perm_importance = permutation_importance(
    rfc, X_test, y_test, n_repeats=10, random_state=42, n_jobs=6)
#forest_importances = pd.Series(result.importances_mean, index=X.columns)
sorted_idx = perm_importance.importances_mean.argsort()
plt.barh(X_test.columns[sorted_idx], perm_importance.importances_mean[sorted_idx])
plt.xlabel("Permutation Importance")
# plt.rcParams["figure.figsize"] = (20,10)
# fig, ax = plt.subplots()
# forest_importances.plot.bar(yerr=result.importances_std[sorted_idx], ax=ax)
# ax.set_title("Feature importances using permutation on full model")
# ax.set_ylabel("Mean accuracy decrease")
# fig.tight_layout()
# plt.show()

# # Feature importance
# ## impurity-based importance.
# importances = rfc.feature_importances_
# std = np.std([
#     tree.feature_importances_ for tree in rfc.estimators_], axis=0)
# forest_importances = pd.Series(importances, index=X.columns)
# plt.rcParams["figure.figsize"] = (20,10)
# fig, ax = plt.subplots()
# forest_importances.plot.bar(yerr=std, ax=ax)
# ax.set_title("Feature importances using MDI")
# ax.set_ylabel("Mean decrease in impurity")
# fig.tight_layout()
shap_values = shap.TreeExplainer(rfc).shap_values(X_test)
shap.summary_plot(shap_values, X_test, plot_type="bar", class_names=rfc.classes_)
#YES
f = plt.figure()
shap.summary_plot(shap_values[0], X_test, class_names=rfc.classes_)
#NO
f = plt.figure()
shap.summary_plot(shap_values[1], X_test, class_names=rfc.classes_)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14579008073817762, 'fraction_numerical': 0.02110726643598616, 'mean_word_length': 6.166115702479339, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '124314', 'license': 'MIT', 'n_tokens_mistral': 3462, 'n_tokens_neox': 3230, 'n_words': 556, 'path': 'notebooks/20_Modeling.ipynb', 'repo_name': 'msenosain/spd-use-of-force_DS4A', 'size': '8670'}