<jupyter_start><jupyter_text>Setup and get data<jupyter_code>import pandas as pd
import psycopg2 as pg
import os
import sys
import pandas.io.sql as pd_sql
import numpy as np
import sqlalchemy
project_dir = str(os.path.dirname(os.path.abspath('')))
print(project_dir)
sys.path.append(project_dir)
import matplotlib.pyplot as plt
from src.pickle.pickle_util import save_pickle, load_pickle

from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score
my_random_state = 72<jupyter_output>/Users/erik/metis/data_hailing<jupyter_text>Start with pickle I saved in notebook 02_eda Alternately, could have loaded it from SQL with the following code:```engine = sqlalchemy.create_engine('postgresql://localhost:5432/data_hailing')query = "SELECT * FROM tnp_trips_2019_pandas WHERE DATE(trip_start_timestamp) = '2019-07-17';"trips_df = pd.read_sql_query(query, engine)```<jupyter_code>file_path = project_dir + '/data/interim/' + 'trips_2019-07-07_from_02_eda.pickle'
# save_pickle(trips_df, file_path)
trips_df = load_pickle(file_path)
trips_df<jupyter_output><empty_output><jupyter_text>Get income data from SQL<jupyter_code>engine = sqlalchemy.create_engine('postgresql://localhost:5432/data_hailing')
query = "SELECT * FROM per_capita_income_chicago_tidy limit 500;"
income_df = pd.read_sql_query(query, engine).dropna()<jupyter_output><empty_output><jupyter_text>Merge data on community area number<jupyter_code>income_df = income_df[['community_area_number', 'per_capita_income_']]

trips_df = trips_df.loc[trips_df.fare > 0.001]
trips_df = trips_df.dropna()
trips_df['community_area_number'] = trips_df['pickup_community_area'].astype(int)

trips_df = trips_df[['community_area_number', 'trip_start_timestamp', 'trip_seconds', 'trip_miles','fare', 'pickup_centroid_latitude','pickup_centroid_longitude']]

df_merge_trip_income = pd.merge(trips_df, income_df, on='community_area_number')
df_merge_trip_income<jupyter_output><empty_output><jupyter_text>Engineer Features  Part of day<jupyter_code>df_merge_trip_income.loc[:, 'part_of_day'] = [hour//4 for hour in df_merge_trip_income.trip_start_timestamp.dt.hour]<jupyter_output><empty_output><jupyter_text>Create surge estimate<jupyter_code>linreg = LinearRegression()
y = df_merge_trip_income.loc[:, 'fare']
X = df_merge_trip_income.loc[:, ['trip_miles', 'trip_seconds']]

linreg.fit(X, y)

pred_fare = linreg.predict(X)
actual_fare = df_merge_trip_income.fare
surge_estimate = actual_fare / pred_fare
df_merge_trip_income['surge_estimate'] = surge_estimate
surge_estimate.describe()<jupyter_output><empty_output><jupyter_text>Setup get X and y<jupyter_code>def get_X(df_merge_trip_income, dummies_on_part_of_day=True):
    X = df_merge_trip_income[['pickup_centroid_latitude', 'pickup_centroid_longitude', 'per_capita_income_', 'part_of_day']]
    if dummies_on_part_of_day:
        X = pd.get_dummies(X, columns=['part_of_day'])
    else:
        X = X.drop['part_of_day']
    return X


def get_y(df_merge_trip_income, surge_cutoff=1.1):
    y = df_merge_trip_income.loc[:, 'surge_estimate'] > surge_cutoff
    return y
X = get_X(df_merge_trip_income)
y = get_y(df_merge_trip_income)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=my_random_state)<jupyter_output><empty_output><jupyter_text>Setup a function for custom cross validation on random forest I selected random forest because it did the best on precision.<jupyter_code>def run_cv_random_forest(X, y, n_estimators=10, predict_proba_threshold=.5):
    #run the CV
    X = np.array(X)
    y = np.array(y)
    X_train, X_test, y_train, y_test = train_test_split(X, y)

    kf = KFold(n_splits=5, shuffle=True, random_state = my_random_state)

    accuracy_scores = []
    precision_scores = []
    recall_scores = []


    for try_ind, val_ind in kf.split(X_train,y_train):

        X_try, y_try = X[try_ind], y[try_ind]
        X_val, y_val = X[val_ind], y[val_ind] 

        rf = RandomForestClassifier(n_estimators=n_estimators, random_state=my_random_state)
        rf.fit(X_try, y_try);

#         y_pred = rf.predict(X_val)
        y_pred = (rf.predict_proba(X_val))[:,1] > predict_proba_threshold
        accuracy_scores.append(accuracy_score(y_val, y_pred))
        precision_scores.append(precision_score(y_val, y_pred))
        recall_scores.append(recall_score(y_val, y_pred))


    print('accuracy_scores: ', accuracy_scores)
    print(f'accuracy_scores: {np.mean(accuracy_scores):.4f} +- {np.std(accuracy_scores):.4f}')

    print('precision_scores: ', precision_scores)
    print(f'precision_scores: {np.mean(precision_scores):.3f} +- {np.std(precision_scores):.3f}')

    print('recall_scores: ', recall_scores)
    print(f'recall_scores: {np.mean(recall_scores):.5f} +- {np.std(recall_scores):.5f}')

    return rf
run_cv_random_forest(X, y, n_estimators=40, predict_proba_threshold=0.45);<jupyter_output>accuracy_scores:  [0.7259603268891529, 0.7181121071389999, 0.7283723944270439, 0.7156640385930806, 0.7193260368663594]
accuracy_scores: 0.7215 +- 0.0048
precision_scores:  [0.35, 0.23178807947019867, 0.29213483146067415, 0.3, 0.3108108108108108]
precision_scores: 0.297 +- 0.038
recall_scores:  [0.0036900369003690036, 0.004516711833785004, 0.0034629728289824187, 0.003434240651233783, 0.005943152454780362]
recall_scores: 0.00421 +- 0.00095<jupyter_text>Run this model on 'holdout' test set of data<jupyter_code>def random_forest_test_set(n_estimators=40, predict_proba_threshold=.45):
    X = get_X(df_merge_trip_income)
    y = get_y(df_merge_trip_income)
    X = np.array(X)
    y = np.array(y)
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=my_random_state)
    
    rf = RandomForestClassifier(n_estimators=n_estimators, random_state=my_random_state)
    rf.fit(X_train, y_train);
    
    accuracy_scores = []
    precision_scores = []
    recall_scores = []
#         y_pred = rf.predict(X_val)
    y_pred = (rf.predict_proba(X_test))[:,1] > predict_proba_threshold
    accuracy_scores.append(accuracy_score(y_test, y_pred))
    precision_scores.append(precision_score(y_test, y_pred))
    recall_scores.append(recall_score(y_test, y_pred))


    print('accuracy_score: ', accuracy_scores[0])

    print('precision_score: ', precision_scores[0])

    print('recall_score: ', recall_scores[0])

    return rf
rf = random_forest_test_set();
rf.feature_importances_
X.columns<jupyter_output><empty_output><jupyter_text>The important columns are: ```'pickup_centroid_latitude': 0.292, 'pickup_centroid_longitude': 0.312,'per_capita_income_': 0.145,'part_of_day_4': 0.145```The rest are less than three percent important     Pickle off model for future use<jupyter_code>file_path = project_dir + '/models/' + 'random_forest_from_02_feat_eng_and_model_notebook.pickle'
save_pickle(rf, file_path)<jupyter_output><empty_output><jupyter_text>Also, save off this data as a csv for use in visualization<jupyter_code>file_path = project_dir + '/data/processed/' + 'df_merge_trip_income_from_03_feat_notebook'
assert not os.path.exists(file_path)
df_merge_trip_income.to_csv(file_path, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11240257076439218, 'fraction_numerical': 0.04922740325447832, 'mean_word_length': 6.602910602910603, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '641924', 'license': 'MIT', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2759, 'n_words': 432, 'path': 'notebooks/03_feat_eng_and_model.ipynb', 'repo_name': 'ekand/rideshare_helper', 'size': '7313'}