<jupyter_start><jupyter_text>Load Data<jupyter_code># Prepare data
demographic = pd.read_csv('../data/processed/demographic.csv')
severity = pd.read_csv('../data/processed/severity.csv', index_col=0)
features = demographic.columns 
X = demographic.astype(np.float64)
y = (severity >= 4).sum(axis=1)
needs_to_label = {0:'no needs', 1:'low_needs', 2:'moderate needs', 3:'high needs', 4:'very high needs'}
labels = ["no needs", "low needs", "moderate needs", "high needs", "very high needs"]
severity_to_needs = {0:0, 1:1, 2:1, 3:2, 4:2, 5:3, 6:3, 7:4, 8:4}
y = np.array([severity_to_needs[i] for i in y])
# Color vector, for illustration purposes
colors = {0:'b', 1:'r', 2:'g', 3:'c', 4:'y'}
y_c = np.array([colors[i] for i in y])
def color_fn(i):
    color_dict = {0:'g', 1:'m', 2:'red', 3:'y', 4:'#FFA500'}
    return color_dict[i]<jupyter_output><empty_output><jupyter_text>Class Balance<jupyter_code>categories, counts = np.unique(y, return_counts=True)
fig, ax = plt.subplots(figsize=(9, 7))
#sb.set(style="whitegrid")
sb.barplot(labels, counts, ax=ax, tick_label=labels)
ax.set(xlabel='Need Categories',
       ylabel='Number of HHs');
#fig.savefig('../reports/figures/class_imbalance.png', dpi=300)<jupyter_output><empty_output><jupyter_text>Heavy class imbalances. Use appropriate scoring metrics/measures.<jupyter_code>from yellowbrick.target import FeatureCorrelation
fig, ax = plt.subplots(figsize=(15, 10))
visualizer = FeatureCorrelation(ax=ax,method='mutual_info-classification',
                                sort=True)
visualizer.fit(X, y, random_state=0)
visualizer.poof()
fig.savefig('../reports/figures/mutual_info.png', dpi=300, bbox_inches = "tight")
fig, ax = plt.subplots(figsize=(15, 10))
visualizer = FeatureCorrelation(labels = X.columns.values)
visualizer.fit(X, y)
visualizer.poof()
fig.savefig('../reports/figures/pearson_corr.png', dpi=300, bbox_inches = "tight")<jupyter_output>/home/muhadriy/.conda/envs/ml/lib/python3.6/site-packages/scipy/stats/stats.py:3038: RuntimeWarning: invalid value encountered in double_scalars
  r = r_num / r_den<jupyter_text>Learning<jupyter_code>from collections import Counter
from sklearn.model_selection import train_test_split
from imblearn.pipeline import make_pipeline
from imblearn.metrics import classification_report_imbalanced
from sklearn.metrics import confusion_matrix
from imblearn.metrics import geometric_mean_score
from sklearn.metrics import balanced_accuracy_score
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, stratify=y)
print('Training target statistics: {}'.format(Counter(y_train)))
print('Testing target statistics: {}'.format(Counter(y_test)))
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.model_selection import StratifiedKFold
from yellowbrick.model_selection import LearningCurve
cv_ = StratifiedKFold(5)
clf = RandomForestClassifier(n_estimators=100, random_state=0)
fig, ax_ = plt.subplots(figsize=(20,10))
viz = LearningCurve(RidgeClassifier(),
    ax=ax_, cv=cv_, train_sizes = np.linspace(0.2, 1., 20),
    scoring='accuracy', n_jobs=-1
)

# Fit and poof the visualizer
viz.fit(X, y)
viz.poof()
fig.savefig('../reports/figures/')<jupyter_output><empty_output><jupyter_text>Random Forests<jupyter_code>from imblearn.ensemble import BalancedRandomForestClassifier
### CrossValidated Parameters ####
params = {'max_depth': 42,
 'max_features': 0.8,
 'min_samples_leaf': 3,
 'min_samples_split': 4,
 'n_estimators': 460}
estimator = BalancedRandomForestClassifier(**params)
viz = LearningCurve(
    estimator, cv=cv_,
    scoring='accuracy', n_jobs=-1
)

# Fit and poof the visualizer
viz.fit(X_train, y_train)
viz.poof()
estimator.fit(X_train, y_train);
y_pred_brf = estimator.predict(X_test)
print('Balanced accuracy: {:.2f}'.format(balanced_accuracy_score(y_test,y_pred_brf)))
print('Classification report: ')
print(classification_report_imbalanced(y_test, y_pred_brf, target_names=labels))
plot_confusion_matrix(y_test, y_pred_brf, classes=np.unique(y_test))<jupyter_output>Balanced accuracy: 0.36
Classification report: 
                       pre       rec       spe        f1       geo       iba       sup

       no needs       0.08      0.57      0.83      0.14      0.69      0.46        51
      low needs       0.34      0.22      0.88      0.27      0.44      0.18       475
 moderate needs       0.57      0.24      0.83      0.34      0.45      0.19      1007
     high needs       0.40      0.34      0.83      0.37      0.53      0.27       523
very high needs       0.02      0.45      0.74      0.03      0.58      0.32        20

    avg / total       0.46      0.27      0.84      0.32      0.47      0.21      2076

Confusion matrix, without normalization<jupyter_text>GridSearchCV<jupyter_code>from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
from imblearn.metrics import geometric_mean_score
from scipy.stats import randint as sp_randint
from scipy.stats import uniform as sp_uniform
cv_ = StratifiedKFold(10)
clf = BalancedRandomForestClassifier()
param_dist = {"n_estimators": np.arange(50,500,10),
              "max_depth": sp_randint(2,50),
              "max_features": [None, 'sqrt', 0.9, 0.8],
              "min_samples_split": sp_randint(2, 11),
              "min_samples_leaf": sp_randint(1,11),
             }
scoring_fns = { 'acc': 'balanced_accuracy', 'gm': make_scorer(geometric_mean_score)}
grid = RandomizedSearchCV(estimator=clf, param_distributions=param_dist, cv=cv_, scoring=scoring_fns,
                            n_iter=100, n_jobs=-1, refit='acc', iid=False, 
                            verbose=3)
grid.fit(X_train, y_train)
y_pred_grid = grid.predict(X_test)
print('Balanced accuracy: {:.2f}'.format(balanced_accuracy_score(y_test,y_pred_grid)))
print('Classification report: ')
print(classification_report_imbalanced(y_test, y_pred_grid, target_names=labels))
plot_confusion_matrix(y_test, y_pred_grid, classes=np.unique(y_test))
#uncomment if you want to avoid running the randomized parameter search
#params = {'max_depth': 42,
# 'max_features': 0.8,
# 'min_samples_leaf': 3,
# 'min_samples_split': 4,
# 'n_estimators': 460}
#estimator = BalancedRandomForestClassifier(**params)
#estimator.fit(X_train, y_train);
estimator = grid.best_estimator_<jupyter_output><empty_output><jupyter_text>SHAP Analysis<jupyter_code>import shap
import warnings
warnings.filterwarnings("ignore")
shap.initjs()
explainer = shap.KernelExplainer(estimator.predict_proba, shap.kmeans(X_train, 5))
shap_values = explainer.shap_values(X_test);
shap.summary_plot(shap_values, X_test, color = color_fn, class_names=needs_to_label)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12454212454212454, 'fraction_numerical': 0.03868131868131868, 'mean_word_length': 4.585924713584288, 'pattern_counts': {'":': 5, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '459508', 'license': 'RSA-MD', 'n_tokens_mistral': 2602, 'n_tokens_neox': 2404, 'n_words': 478, 'path': 'notebooks/Plots.ipynb', 'repo_name': 'mmData/Hack4Good', 'size': '6825'}