<jupyter_start><jupyter_text>Quick Run Script  Provides you with X & y to directly consume<jupyter_code>Sel_Features_DecisionVar = pd.read_pickle('./full/Sel_Features_DecisionVar.pickle', compression='gzip')
exclude_features = [
    'pickup_datetime',
    'pickup_x', 'pickup_y',
    'dropoff_x', 'dropoff_y',
    'Month', 'Hour', 'DayOfWeek',
    'pickup_cluster', 'dropoff_cluster',
    'log_duration', 'duration'
]
features_with_dummies = list(set(Sel_Features_DecisionVar.columns) - set(exclude_features))
selected_features = features_with_dummies
decision_var = [
    'log_duration'
    #'duration'
]
X = Sel_Features_DecisionVar[selected_features]
y = Sel_Features_DecisionVar[decision_var]<jupyter_output><empty_output><jupyter_text>Set seed<jupyter_code>seed = 140
np.random.seed(seed)<jupyter_output><empty_output><jupyter_text>Train - Test Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)<jupyter_output><empty_output><jupyter_text>Data Pre-Processing & Feature Engineering<jupyter_code>pd.options.display.max_rows = 250
gotham_cabs_train = pd.read_csv('Train.csv')
quantiles
quantile = 0.05
quantiles = gotham_cabs_train.quantile([quantile, 1-quantile])
gotham_cabs_train_fil = gotham_cabs_train.copy()
gotham_cabs_train_fil = gotham_cabs_train_fil.loc[(gotham_cabs_train.pickup_x > quantiles.loc[quantile].pickup_x) & (gotham_cabs_train.pickup_x < quantiles.loc[1-quantile].pickup_x)]
gotham_cabs_train_fil = gotham_cabs_train_fil.loc[(gotham_cabs_train.pickup_y > quantiles.loc[quantile].pickup_y) & (gotham_cabs_train.pickup_y < quantiles.loc[1-quantile].pickup_y)]
gotham_cabs_train_fil = gotham_cabs_train_fil.loc[(gotham_cabs_train.dropoff_x > quantiles.loc[quantile].dropoff_x) & (gotham_cabs_train.dropoff_x < quantiles.loc[1-quantile].dropoff_x)]
gotham_cabs_train_fil = gotham_cabs_train_fil.loc[(gotham_cabs_train.dropoff_y > quantiles.loc[quantile].dropoff_y) & (gotham_cabs_train.dropoff_y < quantiles.loc[1-quantile].dropoff_y)]
gotham_cabs_train_fil = gotham_cabs_train_fil.loc[(gotham_cabs_train.duration > quantiles.loc[quantile].duration) & (gotham_cabs_train.duration < quantiles.loc[1-quantile].duration)]
gotham_cabs_train_fil.head()
gotham_cabs_train_fil['Month'] = gotham_cabs_train_fil['pickup_datetime'].apply(lambda s: dt.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').month)
gotham_cabs_train_fil['Hour'] = gotham_cabs_train_fil['pickup_datetime'].apply(lambda s: dt.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').hour)
gotham_cabs_train_fil['DayOfWeek'] = gotham_cabs_train_fil['pickup_datetime'].apply(lambda s: dt.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').isoweekday())
def minDistance(_record):
    return sp.spatial.distance.euclidean([_record['pickup_x'], _record['pickup_y']], [_record['dropoff_x'], _record['dropoff_y']])
def manhattanDistance(_record):
    return sp.spatial.distance.cityblock([_record['pickup_x'], _record['pickup_y']], [_record['dropoff_x'], _record['dropoff_y']])
def haversineDistance(_record):
    return haversine((_record['pickup_x'], _record['pickup_y']), (_record['dropoff_x'], _record['dropoff_y']))
gotham_cabs_train_fil['MinDistance'] = gotham_cabs_train_fil.apply(lambda s: minDistance(s), axis=1)
gotham_cabs_train_fil['HaversineDistance'] = gotham_cabs_train_fil.apply(lambda s: haversineDistance(s), axis=1)
gotham_cabs_train_fil['ManhattanDistance'] = gotham_cabs_train_fil.apply(lambda s: manhattanDistance(s), axis=1)
gotham_cabs_train_fil['log_duration'] = gotham_cabs_train_fil.apply(lambda s: 0 if s.duration==0 else np.log(s.duration), axis=1)
gotham_cabs_train_fil.to_pickle('gotham_cabs_train_fil.pickle')<jupyter_output><empty_output><jupyter_text>10% Random Sample for Analysis<jupyter_code>gotham_cabs_train_fil = pd.read_pickle('gotham_cabs_train_fil.pickle')
sampled_gotham_city = gotham_cabs_train_fil.sample(frac=1, replace=False)<jupyter_output><empty_output><jupyter_text>K-Means<jupyter_code>sampled_gotham_city.columns<jupyter_output><empty_output><jupyter_text>Pick-Up Clustering<jupyter_code>#pickup_df = X[['pickup_x', 'pickup_y']]
# kmeans = KMeans(n_clusters=60).fit(pickup_df)
# pickup_cluster = kmeans.predict(pickup_df)
#X['pickup_cluster'] = pickup_cluster
#plt.scatter(X['pickup_x'], X['pickup_y'], c=pickup_cluster, s=10)<jupyter_output><empty_output><jupyter_text>Drop-Up Clustering<jupyter_code>#dropoff_df = X[['dropoff_x', 'dropoff_y']]
#kmeans = KMeans(n_clusters=60).fit(dropoff_df)
#dropoff_cluster = kmeans.predict(dropoff_df)
#X['dropoff_cluster'] = dropoff_cluster
#plt.scatter(X['dropoff_x'], X['dropoff_y'], c=dropoff_cluster, s=10)<jupyter_output><empty_output><jupyter_text>Pick-Drop Joint Clustering<jupyter_code>#x_coord = pd.concat([sampled_gotham_city['pickup_x'], sampled_gotham_city['dropoff_x']], axis=0)
#y_coord = pd.concat([sampled_gotham_city['pickup_y'], sampled_gotham_city['dropoff_y']], axis=0)
#xy_coord = pd.concat([x_coord, y_coord], axis=1)
#kmeans = KMeans(n_clusters=600).fit(xy_coord)
#from sklearn.externals import joblib
#joblib.dump(kmeans, 'kmeans_600.pkl') 
kmeans = joblib.load('kmeans_600.pkl') 
pickup_df = sampled_gotham_city[['pickup_x', 'pickup_y']]
pickup_cluster = kmeans.predict(pickup_df)
sampled_gotham_city['pickup_cluster'] = pickup_cluster
dropoff_df = sampled_gotham_city[['dropoff_x', 'dropoff_y']]
dropoff_cluster = kmeans.predict(dropoff_df)
sampled_gotham_city['dropoff_cluster'] = dropoff_cluster
pickup_cluster = kmeans.predict(xy_coord)
plt.scatter(xy_coord[0], xy_coord[1], c=pickup_cluster, s=10)
sampled_gotham_city.to_pickle('sampled_gotham_city.pickle')<jupyter_output><empty_output><jupyter_text>Into-Categorical<jupyter_code>sampled_gotham_city = pd.read_pickle('sampled_gotham_city.pickle')
sampled_gotham_city.columns
# sampled_gotham_city['Month'] = np_utils.to_categorical(sampled_gotham_city['Month'])
# sampled_gotham_city['Hour'] = np_utils.to_categorical(sampled_gotham_city['Hour'])
# sampled_gotham_city['DayOfWeek'] = np_utils.to_categorical(sampled_gotham_city['DayOfWeek'])

# sampled_gotham_city['pickup_cluster'] = np_utils.to_categorical(sampled_gotham_city['pickup_cluster'])
# sampled_gotham_city['dropoff_cluster'] = np_utils.to_categorical(sampled_gotham_city['dropoff_cluster'])
df_with_dummies = pd.get_dummies(sampled_gotham_city, columns = ['Month', 'Hour', 'DayOfWeek', 'pickup_cluster', 'dropoff_cluster'])
df_with_dummies.to_pickle('sampled_gotham_city_cat.pickle')
#sampled_gotham_city.to_pickle('sampled_gotham_city_cat.pickle')<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>sampled_gotham_city_cat = pd.read_pickle('./full/sampled_gotham_city_cat.pickle')
exclude_features = [
    'pickup_datetime',
    'pickup_x', 'pickup_y',
    'dropoff_x', 'dropoff_y',
    'Month', 'Hour', 'DayOfWeek',
    'pickup_cluster', 'dropoff_cluster',
    'log_duration', 'duration'
]
features_with_dummies = list(set(sampled_gotham_city_cat.columns) - set(exclude_features))
features_with_dummies
# selected_features = [
#     #'pickup_datetime',
#     #'pickup_x', 'pickup_y',
#     #'dropoff_x', 'dropoff_y',
#     'Month', 'Hour', 'DayOfWeek',
#     'MinDistance',
#     'HaversineDistance', 
#     'ManhattanDistance',
#     'pickup_cluster', 'dropoff_cluster'
# ]
selected_features = features_with_dummies
decision_var = [
    'log_duration'
    #'duration'
]
df = sampled_gotham_city_cat.copy(deep=True)
X = df[selected_features]
y = df[decision_var]<jupyter_output><empty_output><jupyter_text>Normalization of Selected Variables<jupyter_code>Sel_Features_DecisionVar = pd.concat([X,y], axis=1)
missing_df = Sel_Features_DecisionVar.isnull().sum(axis=0).reset_index()
missing_df.columns = ['variable', 'missing values']
missing_df['filling factor (%)']=(df.shape[0]-missing_df['missing values'])/df.shape[0]*100
missing_df.sort_values('filling factor (%)').reset_index(drop = True)
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
print(scaler.fit(Sel_Features_DecisionVar))
Sel_Features_DecisionVar[list(Sel_Features_DecisionVar.columns)] = scaler.transform(Sel_Features_DecisionVar)
Sel_Features_DecisionVar
#Sel_Features_DecisionVar.to_pickle('Sel_Features_DecisionVar.pickle', compression='gzip')
Sel_Features_DecisionVar = pd.read_pickle('./full/Sel_Features_DecisionVar.pickle', compression='gzip')
X = Sel_Features_DecisionVar[selected_features]
y = Sel_Features_DecisionVar[decision_var]
print(X.shape, y.shape)<jupyter_output>(876761, 1209) (876761, 1)<jupyter_text>Test - Train Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
X_train.head()
y_train.head()<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>import numpy
from keras.datasets import mnist
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers.convolutional import Conv2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from keras import backend as K
import numpy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# def baseline_model():
#     # create model
#     model = Sequential()
#     model.add(Dense(1205, input_dim=1209, kernel_initializer='normal', activation='relu'))
#     model.add(Dense(1, kernel_initializer='normal'))
#     # Compile model
#     model.compile(loss='mean_squared_error', optimizer='adam')
#     return model
# estimator = KerasRegressor(build_fn=baseline_model, epochs=2, batch_size=1000, verbose=0)
seed = 140
np.random.seed(seed)
# kfold = KFold(n_splits=10, random_state=seed)
# results = cross_val_score(estimator, X, y, cv=kfold)
# print("Results: %.2f (%.2f) MSE" % (results.mean(), results.std()))
def larger_model():
    # create model
    model = Sequential()
    model.add(Dense(1205, input_dim=1209, activation='sigmoid'))
    #model.add(Conv2D(20, (7, 7), input_shape=(10), activation='relu'))
    #model.add(MaxPooling2D(pool_size=(4, 4)))
    #model.add(Dropout(0.25))
    #model.add(Flatten())
    model.add(Dense(10000, activation='sigmoid'))
    model.add(Dense(1296, activation='hard_sigmoid'))
    model.add(Dense(36, activation='sigmoid'))
    model.add(Dense(1, activation='sigmoid'))
    # Compile model
    model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mse', 'mae', 'mape', 'cosine'])
    return model
# build the model
model = larger_model()
# Fit the model: For more accurate results increase epochs to 10
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=5, batch_size=2000)
# Final evaluation of the model
#model.save('nn_model.h5')
estimator = KerasRegressor(build_fn=larger_model, epochs=3, batch_size=2000, verbose=1)
kfold = KFold(n_splits=10, random_state=seed)
results = cross_val_score(estimator, X, y, cv=kfold, n_jobs=3)
print("Results: %.2f (%.2f) MSE" % (results.mean(), results.std()))
# # evaluate model with standardized dataset
# np.random.seed(seed)
# estimators = []
# estimators.append(('standardize', StandardScaler()))
# estimators.append(('mlp', KerasRegressor(build_fn=larger_model, epochs=3, batch_size=2000, verbose=0)))
# pipeline = Pipeline(estimators)
# kfold = KFold(n_splits=10, random_state=seed)
# results = cross_val_score(pipeline, X, y, cv=kfold, n_jobs=3)
# print("Standardized: %.2f (%.2f) MSE" % (results.mean(), results.std()))<jupyter_output><empty_output><jupyter_text>Random Forest Regressor<jupyter_code>regr = RandomForestRegressor(max_depth=6)
regr.fit(X_train, y_train.values.ravel())
regr.feature_importances_
y_pred = regr.predict(X_test)
mean_squared_error(y_test, y_pred)
kfold = KFold(n_splits=10, random_state=seed)
results = cross_val_score(regr, X, y.values.ravel(), cv=kfold)
print("Results: %.2f (%.2f) MSE" % (results.mean(), results.std()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13934290903087337, 'fraction_numerical': 0.01419844807660558, 'mean_word_length': 8.94663382594417, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '311154', 'license': 'MIT', 'n_tokens_mistral': 4837, 'n_tokens_neox': 4642, 'n_words': 607, 'path': 'Preprocessing to Model Building.ipynb', 'repo_name': 'vSanjeev/Gotham-City-Cabs', 'size': '12114'}