<jupyter_start><jupyter_text>Model SelectionPreliminary model assessment showed performance of classification algorithms performing better than regression.   Thus, for the purposes of this notebook, only classification algorithms are explored.<jupyter_code>%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
import pandas as pd
import matplotlib.pyplot as plt
import sqlite3

import pickle
import joblib

from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score

from sklearn.ensemble import BaggingClassifier, ExtraTreesClassifier, RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF

from sklearn.preprocessing import OneHotEncoder, LabelEncoder

from yellowbrick.classifier import ClassificationReport
pd.set_option('display.max_columns', 100)
pd.set_option('display.max_rows', 100)
conn = sqlite3.connect('../data/crime_census_weather_tod.db')
cursor = conn.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
print(cursor.fetchall())
conn = sqlite3.connect('../data/crime_census_weather_tod.db')
df = pd.read_sql_query("SELECT * FROM all_crimes", conn)
conn.close()
df['crime_rate_cat'].unique()
cats = list(df['crime_rate_cat'].unique())
df.head(500)
df_d = df
#This doesn't work, so why did it work in the standardize_data notebook
le = preprocessing.LabelEncoder()
bg_fit = le.fit(df['BLOCK_GROUP'])
df_d['bg_cat'] = bg_fit.transform(df_d['BLOCK_GROUP'])
print(df_d.columns, len(df_d.columns))

le = preprocessing.LabelEncoder()
bg_fit = le.fit(df_d['BLOCK_GROUP'])
df_d['bg_cat'] = bg_fit.transform(df_d['BLOCK_GROUP'])
print(df_d.columns, len(df_d.columns))<jupyter_output>Index(['level_0', 'BLOCK_GROUP', 'year', 'month', 'day', 'tod_cat', 'tod_num',
       'crime_counts', 'apparent_temp', 'cloud_cover', 'dew_point', 'humidity',
       'percip_intensity', 'percip_probability', 'pressure', 'temperature',
       'uv_index', 'visibility', 'wind_bearing', 'wind_gust', 'wind_speed',
       'weekday', 'index', 'TotalPop', 'TPopMargin', 'UnWgtSampleCtPop',
       'PerCapitaIncome', 'PerCapIncMargin', 'MedianHouseholdInc',
       'MedHouseholdIncMargin', 'MedianAge', 'MedianAgeMargin', 'HousingUnits',
       'HousingUnitsMargin', 'UnweightedSampleHousingUnits', 'State', 'County',
       'Tract', 'BlockGroup', 'census_year', 'crime_rate', 'crs',
       'crime_rate_cat', 'bg_cat'],
      dtype='object') 44<jupyter_text>Saving bg_cat encoder<jupyter_code>bg_cat_encoder = 'bg_cat_encoder.sav'
pickle.dump(bg_fit, open(bg_cat_encoder, 'wb'))
df_m = df_d.dropna(how='any')
df_m.head()
df_m.tod_cat.unique()
print(len(df_d),len(df_m), len(df)-len(df_m))
feature_cols = [
    'year', 'month', 'day', 'tod_num',
    'cloud_cover', 'dew_point', 'humidity', 'percip_intensity',
    'percip_probability', 'pressure', 'temperature', 'uv_index',
    'visibility', 'wind_bearing', 'wind_gust', 'wind_speed',
    'PerCapitaIncome', 'MedianHouseholdInc', 'MedianAge', 'HousingUnits',
    'weekday', 'bg_cat'
]

target_cols = 'crime_rate_cat'
fig = plt.figure(figsize=(25,9))
ax1 = fig.add_subplot(1,1,1)

df_m[feature_cols].boxplot(ax=ax1, rot=60)
scaler = preprocessing.StandardScaler()
#robust = preprocessing.RobustScaler()
#minmax = preprocessing.MinMaxScaler()

scaler_df = pd.DataFrame(scaler.fit_transform(df_m[feature_cols]), columns=feature_cols)
#robust_df = pd.DataFrame(robust.fit_transform(df_m[feature_cols]), columns=feature_cols)
#minmax_df = pd.DataFrame(minmax.fit_transform(df_m[feature_cols]), columns=feature_cols)


scaler.fit_transform(df_m[feature_cols])
test = scaler.fit(df_m[feature_cols])
test
fig = plt.figure(figsize=(25,20))

ax1 = fig.add_subplot(2,2,1)
ax2 = fig.add_subplot(2,2,2)
ax3 = fig.add_subplot(2,2,3)

scaler_df.boxplot(ax=ax1)
robust_df.boxplot(ax=ax2)
minmax_df.boxplot(ax=ax3)

ax1_labels = ax1.get_xticklabels()
ax2_labels = ax2.get_xticklabels()
ax3_labels = ax3.get_xticklabels()

ax1.set_xticklabels(ax1_labels,rotation=45)
ax2.set_xticklabels(ax2_labels,rotation=45)
ax3.set_xticklabels(ax3_labels,rotation=45)

ax1.title.set_text('Scalar')
ax2.title.set_text('Robust')
ax3.title.set_text('Min-Max')

plt.show()
features_scaled = scaler_df
target = df_m[target_cols]
# Small sample to make sure the models work
X_train, X_test, y_train, y_test = train_test_split(features_scaled, target, test_size=30000, train_size=30000)
from yellowbrick.features import FeatureImportances

model = RandomForestClassifier(n_estimators=10)
viz = FeatureImportances(model, size=(900, 720))

viz.fit(features_scaled, target)
viz.poof()
features_scaled.head()
list(features_scaled.columns)
features_scaled_2 = features_scaled.drop(['PerCapitaIncome', 'MedianHouseholdInc', 'MedianAge', 'HousingUnits'], axis=1)
from yellowbrick.features import FeatureImportances



model = RandomForestClassifier(n_estimators=10)
viz = FeatureImportances(model, size=(900, 720))

viz.fit(features_scaled_2, target)
viz.poof()
features_scaled_3 = features_scaled_2.drop(['percip_probability', 'dew_point', 'pressure', 'wind_speed', 
                                           'wind_bearing', 'wind_gust', 'humidity'], axis=1)

from yellowbrick.features import FeatureImportances



model = RandomForestClassifier(n_estimators=10)
viz = FeatureImportances(model, size=(900, 720))

viz.fit(features_scaled_3, target)
viz.poof()

from sklearn.feature_selection import RFECV
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold

X_train, X_test, y_train, y_test = train_test_split(features_scaled_3, target, test_size=30000, train_size=30000)

rmf = RandomForestClassifier()
rfecv = RFECV(estimator=rmf, step=1, cv=StratifiedKFold(9), scoring='accuracy', verbose = 10, n_jobs = -1)
rfecv.fit(X_train , y_train)
optimal_features = rfecv.n_features_

print("Optimal number of features : %d" % rfecv.n_features_)
plt.figure(figsize=(15, 10))
plt.xlabel("Number of Features Selected")
plt.ylabel("Cross Validation Score")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()<jupyter_output><empty_output><jupyter_text>Comparing various classification models<jupyter_code># Adapted from 506 Lab:
def score_model(Xtr, Xte, ytr, yte, estimator):
    """
    Test various estimators.
    """ 
    
    model = estimator
    
    # Instantiate the classification model and visualizer
    model.fit(Xtr, ytr)  
    
    expected  = yte
    predicted = model.predict(Xte)
    
    # Compute and return F1 (harmonic mean of precision and recall)
    print("{}: {}".format(estimator.__class__.__name__, f1_score(expected, predicted, average = 'micro')))
    print('Accuracy score = %0.3f' % accuracy_score(expected, predicted))
    print(confusion_matrix(expected, predicted))
# You get a model!  And you get a model!  Everyone gets a model!

models = [
    KNeighborsClassifier(), 
    BaggingClassifier(), ExtraTreesClassifier(n_estimators=100), 
    RandomForestClassifier(n_estimators=100), ExtraTreesClassifier(),
    DecisionTreeClassifier()
]

for model in models:
    score_model(X_train, X_test, y_train, y_test, model)<jupyter_output>KNeighborsClassifier: 0.7089333333333333
Accuracy score = 0.709
[[   29     0     8   477   103]
 [    0   264   125    86     0]
 [    1    49  1694  2634     5]
 [   49     7   891 18550   634]
 [   59     0    34  3570   731]]
BaggingClassifier: 0.9536333333333333
Accuracy score = 0.954
[[  314     0     0   145   158]
 [    0   470     4     1     0]
 [    0    19  4336    26     2]
 [   26    13   266 19769    57]
 [   65     0     8   601  3720]]
ExtraTreesClassifier: 0.9402666666666667
Accuracy score = 0.940
[[  276     0     0   170   171]
 [    0   468     3     4     0]
 [    0    18  4203   162     0]
 [    4    13   249 19845    20]
 [   15     0     7   956  3416]]
RandomForestClassifier: 0.9535
Accuracy score = 0.954
[[  290     0     0   155   172]
 [    0   470     3     2     0]
 [    0    19  4324    40     0]
 [    0    13   255 19811    52]
 [    8     0     7   669  3710]]
ExtraTreesClassifier: 0.9218333333333333
Accuracy score = 0.922
[[  266     0     0   197   1[...]<jupyter_text>Visualize precision, recall, and f1 scores<jupyter_code># Adapted from 506 lab
def visualize_model(Xtr, Xte, ytr, yte, estimator):
    """
    Test various estimators.
    """ 
    #y = LabelEncoder().fit_transform(y)
    #model = Pipeline([
    #     ('one_hot_encoder', OneHotEncoder()), 
    #     ('estimator', estimator)
    #])

    # Instantiate the classification model and visualizer
    visualizer = ClassificationReport(
        model, classes=cats, 
        cmap="Reds", size=(600, 360)
    )
    visualizer.fit(Xtr, ytr)  
    visualizer.score(Xte, yte)
    visualizer.poof()  

#for model in models:
#    visualize_model(X_train, X_test, y_train, y_test, model)
model = BaggingClassifier()
model.fit(X_train, y_train)

expected = y_test
predicted = model.predict(X_test)

print('Bagging Classifier')
print('Accuracy score = %0.3f' % accuracy_score(expected, predicted))
print("{}: {}".format(model.__class__.__name__, f1_score(expected, predicted, average = None)))
confusion_matrix(expected, predicted, labels=cats)
cats
features_final = features_scaled_3[['bg_cat', 'weekday', 'tod_num', 'uv_index', 'temperature', 'day']].copy()

X_train, X_test, y_train, y_test = train_test_split(features_final, target, test_size=.3)

for model in models:
    visualize_model(X_train, X_test, y_train, y_test, model)
features_scaled.head()
# Small sample to make sure the models work
X_train, X_test, y_train, y_test = train_test_split(features_scaled, target, test_size=30000, train_size=30000)
#This was the prelim features that were initially identified
features_subset = features_scaled[['month', 'day', 'tod_num', 'humidity', 'temperature', 'uv_index', 
                     'weekday', 'bg_cat']].copy()<jupyter_output><empty_output><jupyter_text>monthdaytod_cathumiditytemperatureuv_indexweekdaybg_cat<jupyter_code># You get a model!  And you get a model!  Everyone gets a model!

# Small sample to make sure the models work
X_train, X_test, y_train, y_test = train_test_split(features_subset, target, test_size=30000, train_size=30000)

models = [
    KNeighborsClassifier(), 
    BaggingClassifier(), ExtraTreesClassifier(n_estimators=100), 
    RandomForestClassifier(n_estimators=100), ExtraTreesClassifier(),
    DecisionTreeClassifier()
]

for model in models:
    score_model(X_train, X_test, y_train, y_test, model)
filename = '../JobLib/BaggingClassifier.sav'
joblib.dump(model, filename)
filename = '../JobLib/BaggingClassifier.sav'
loaded_model = joblib.load(filename)
result = loaded_model.score(X_test, y_test)
print(result)
result = model.predict(X.iloc[[455]])
result[0]
# Each feature of this dataset is an 8x8 pixel image of a handwritten number.
# Digits.data converts these 64 pixels into a single array of features

from yellowbrick.classifier import ConfusionMatrix

X = features_final
y = target

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size =0.2, random_state=11)

model = BaggingClassifier()

# The ConfusionMatrix visualizer taxes a model
#cm = ConfusionMatrix(model, classes=[0,1,2,3,4,5,6,7,8,9])
cm = ConfusionMatrix(model, classes=cats)

# Fit fits the passed model. This is unnecessary if you pass the visualizer a pre-fitted model
cm.fit(X_train, y_train)

# To create the ConfusionMatrix, we need some test data. Score runs predict() on the data
# and then creates the confusion_matrix from scikit-learn.
cm.score(X_test, y_test)

# How did we do?
cm.poof()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11773597581457844, 'fraction_numerical': 0.04383607658716829, 'mean_word_length': 5.045177664974619, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '638253', 'license': 'MIT', 'n_tokens_mistral': 4538, 'n_tokens_neox': 4167, 'n_words': 926, 'path': 'notebooks/DRAFT-ML-notebooks/ModelSelectionEval_v4.ipynb', 'repo_name': 'georgetown-analytics/DC-Criminalistics', 'size': '11908'}