<jupyter_start><jupyter_text>Data Prep<jupyter_code>data2 = pd.DataFrame(index=data.index.copy())
data2 = data[['Name', 'City', 'Deposit', 'ClosestPSDistance',
       'MeanPSDistance', 'PSCount', 'Take', 'PDistance', 'Officers1000',
       'FFLCount', 'AvgRating', 'Target', 'Population', 'CrimeRate1000' ]].copy()
data2['CrimeRate1000'].mean()
data2['Population'].mean()
data2.isnull().sum()
values = {'CrimeRate1000': data2['CrimeRate1000'].mean(), 'Population': data2['Population'].mean(), 'AvgRating' : data2['AvgRating'].mean()}
data2.fillna(value=values, inplace=True)
#data2.dropna(axis=0, inplace=True)
data2.shape
# Map target class 1 - 5 (70% success rate and up) to 1 and the rest to 0 
data2['target_bin'] = data2.Target.map({ 0:0, 1:1, 2:1, 3:1, 4:1, 5:1 })
data2.target_bin[data2.target_bin != 0].sum() #total target<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>plt.figure(figsize = (15,10))
sns.heatmap(data2.corr(), annot=True)
#Distance, P(Caught by Distance), Target correlation
g = sns.pairplot(data2[['ClosestPSDistance',  'FFLCount', 'Officers1000', 'Take', 'Population', 'AvgRating', 'CrimeRate1000', 'target_bin']], diag_kind="hist", kind="reg")
for ax in g.axes.flat:
    plt.setp(ax.get_xticklabels(), rotation=45)<jupyter_output><empty_output><jupyter_text>RandomForest with GridSearch for best Features, Max Depth and Min Leaf<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
feature_cols = ['ClosestPSDistance',
       'MeanPSDistance', 'PSCount', 'Take', 'PDistance', 'Officers1000',
       'FFLCount', 'AvgRating', 'Population', 'CrimeRate1000']
X = data2[feature_cols]
y = data2['target_bin']
#Use RandomForest
rf = RandomForestClassifier(random_state=1)
max_depth_range = range(1, 21)
param_grid = dict(max_depth=list(max_depth_range))
grid = GridSearchCV(rf, param_grid, cv=10, scoring='roc_auc')
grid.fit(X, y)

grid_mean_scores = grid.cv_results_['mean_test_score']

# plot the results
plt.plot(max_depth_range, grid_mean_scores)
plt.xlabel('Value of max_depth')
plt.ylabel('Cross-Validated AUC')
print (grid.best_score_)
print (grid.best_params_)
print (grid.best_estimator_)
pd.DataFrame({'feature':feature_cols, 'importance':grid.best_estimator_.feature_importances_}).sort_values(by=['importance'])<jupyter_output><empty_output><jupyter_text>Remove Features<jupyter_code># Remove features: MeanPSDistance, PSCount
# Use the rest
feature_cols = ['ClosestPSDistance', 'Take', 'PDistance', 'Officers1000',
       'FFLCount', 'AvgRating', 'Population', 'CrimeRate1000']
X = data2[feature_cols]
y = data2['target_bin']

rf = RandomForestClassifier(random_state=1)
max_depth_range = range(1, 21)
param_grid = dict(max_depth=list(max_depth_range))
grid = GridSearchCV(rf, param_grid, cv=10, scoring='roc_auc')
grid.fit(X, y)

grid_mean_scores = grid.cv_results_['mean_test_score']

# plot the results
plt.plot(max_depth_range, grid_mean_scores)
plt.xlabel('Value of max_depth')
plt.ylabel('Cross-Validated AUC')
print (grid.best_score_)<jupyter_output>0.958712429183<jupyter_text>Best Depth and Min Leaf<jupyter_code># Use Grid Search Cross Validation for best depth AND min leaf
max_depth_range = range(1, 21)
leaf_range = range(1, 11)
param_grid = dict(max_depth=list(max_depth_range), min_samples_leaf=list(leaf_range))
grid = GridSearchCV(rf, param_grid, cv=10, scoring='roc_auc')
grid.fit(X, y)
print (grid.best_score_)
print (grid.best_params_)
print (grid.best_estimator_)<jupyter_output>0.969067668818
{'max_depth': 8, 'min_samples_leaf': 2}
RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
            max_depth=8, max_features='auto', max_leaf_nodes=None,
            min_impurity_decrease=0.0, min_impurity_split=None,
            min_samples_leaf=2, min_samples_split=2,
            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
            oob_score=False, random_state=1, verbose=0, warm_start=False)<jupyter_text>Use the max_depth and min_samples_leaf for the model<jupyter_code>rf2 = RandomForestClassifier(random_state=1, max_depth=8, min_samples_leaf=2)
rf2.fit(X,y)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
#Check the model with prediction
y_pred = rf2.predict(X_test)
metrics.accuracy_score(y_test, y_pred)
from sklearn.tree import export_graphviz
idx = 0
for tree in rf2.estimators_:
    export_graphviz(tree, out_file='tree_banks_' + str(idx) + '.dot', feature_names=feature_cols)
    idx = idx + 1
! dot -Tpng tree_banks_1.dot -o tree_banks_1.png
from IPython.display import Image
from IPython.display import display

display(Image('tree_banks_1.png'))<jupyter_output><empty_output><jupyter_text>Save Model to Pickle<jupyter_code>try:
    import cPickle as pickle
except ImportError:
    import pickle
## Save the RandomForestClassifier model
out_s = open('targetbanks_randomforestclassifier.pkl', 'wb')
pickle.dump(rf2, out_s)
out_s.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1370110330992979, 'fraction_numerical': 0.033099297893681046, 'mean_word_length': 7.508532423208191, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '504961', 'license': 'MIT', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1823, 'n_words': 344, 'path': 'Modeling.ipynb', 'repo_name': 'hhalim/TargetBanksJupyter', 'size': '4985'}