<jupyter_start><jupyter_text>chh-OLA<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
%matplotlib inline
pd.set_option('display.float_format', lambda x: '%.3f' % x)
url='https://raw.githubusercontent.com/roopbiswas/chh-ola/master/train.csv?token=<KEY>'
df=pd.read_csv(url,encoding='utf7')
df['stored_flag'] = df['stored_flag'].apply(lambda x: 0 if x=='N' else 1)
df_mod = df
df.info()
df.isnull().sum()
df.dropna(axis=0,inplace=True)
df.dtypes
df['total_amount'] = df['total_amount'].astype(float)
df['driver_tip'] = df['driver_tip'].astype(float)
df['mta_tax'] = df['mta_tax'].astype(float)
df['pickup_loc'] = df['pickup_loc'].astype(int)
df['toll_amount'] = df['toll_amount'].astype(float)
df['extra_charges'] = df['extra_charges'].astype(float)
df['improvement_charge'] = df['improvement_charge'].astype(float)
df['vendor_id'] = df['improvement_charge'].astype(float)
df.head()
df.describe()
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
y = df['total_amount']
X_train, X_other, y_train, y_other = train_test_split(X, y, test_size=0.4, random_state=42)
X_cv, X_test, y_cv, y_test = train_test_split(X_other, y_other, test_size=0.5, random_state=42)



print("Train dataset size: ", X_train.shape[0])
print("CV size: ", X_cv.shape[0])
print("Test size: ", X_test.shape[0])
def model_results(X_train, y_train, X_test, y_test, model):
    """Print model parameters of RMSE and R-square on training and testing sets.
    """
    
    y_train_pred = model.predict(X_train)
    y_test_pred = model.predict(X_test)
    print("----Training Data results----")
    print("RMSE: ${:.1f}".format(mean_squared_error(y_train, y_train_pred)**0.5))
    print("R2: {:.2f}\n".format(r2_score(y_train, y_train_pred)))    
    print("----Test Data results----")
    print("RMSE: ${:.1f}".format(mean_squared_error(y_test, y_test_pred)**0.5))
    print("R2: {:.2f}\n".format(r2_score(y_test, y_test_pred)))
from sklearn.linear_model import LinearRegression


lm = LinearRegression().fit(X_train,y_train)

model_results(X_train, y_train, X_test, y_test, lm)
from sklearn.svm import SVR


reg = SVR().fit(X_train,y_train)
model_results(X_train, y_train, X_test, y_test, reg)
# pd.plotting.scatter_matrix(frame = df[[ 'total_amount' , 'pickup_loc' , 'drop_loc' , 'mta_tax' , 'distance' , 'pickup_time' , 'drop_time' , 'num_passengers' , 'toll_amount' ]] , figsize = (7,7))
df.dtypes
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
z =df['drop_time'] - df['pickup_time']
X['travel_time'] = [z1.total_seconds() for z1 in z]
y = df['total_amount']
X_train, X_other, y_train, y_other = train_test_split(X, y, test_size=0.4, random_state=42)
X_cv, X_test, y_cv, y_test = train_test_split(X_other, y_other, test_size=0.5, random_state=42)



print("Train dataset size: ", X_train.shape[0])
print("CV size: ", X_cv.shape[0])
print("Test size: ", X_test.shape[0])
X.head()
from sklearn.linear_model import LinearRegression


lm = LinearRegression().fit(X_train,y_train)
model_results(X_train, y_train, X_test, y_test, lm)
def plot_scatter(x,y,label,x_label,y_label):
  plt.scatter(x, y, c="g", alpha=0.5, label=label)
  plt.xlabel(x_label)
  plt.ylabel(y_label)
  plt.legend(loc='upper right')
  plt.title(label)
  # plt.xticks(np.arange(0, 1, step=0.2))
  plt.figure(figsize=(20,10))
  plt.show()
plot_scatter(df['distance'][:100000],df['total_amount'][:100000],"distance vs. total_amount","distance","total_amount")
df.describe()
print(f"There are {len(df[df['total_amount'] < 0])} negative entries of total_amount.")
print(f"There are {len(df[df['total_amount'] == 0])} 0 entries of total_amount.")
print(f"There are {len(df[df['total_amount'] > 600])} entries of total_amount greater than 600.")
print(f"There are {len(df[df['vendor_id'] < 0])} negative entries of vendor_id.")
print(f"There are {len(df[df['driver_tip'] < 0])} negative entries of driver_tip.")
print(f"There are {len(df[df['mta_tax'] < 0])} negative entries of mta_tax.")<jupyter_output>There are 521 negative entries of total_amount.
There are 203 0 entries of total_amount.
There are 9 entries of total_amount greater than 600.
There are 521 negative entries of vendor_id.
There are 12 negative entries of driver_tip.
There are 505 negative entries of mta_tax.<jupyter_text>*The above plot along with values from describe show that there are outlier and meaningless points which are very less in number..**   Neglecting negative values from amounts (removing outliers of total_amount corrects other attributes)*   Assuming 600 to be the upper cap of fare*<jupyter_code>df=df[df['total_amount'].between(left=0.1,right=600)]
# df=df[df['vendor_id']>=0]
# df=df[df['toll_amount']>=0]
# df=df[df['mta_tax']>=0]
# df=df[df['driver_tip']>=0]
# df=df[df['improvement_charge']>=0]
# df=df[df['extra_charges']>=0]
df.describe()
cols = list(df.columns)
cols.remove('pickup_time')
cols.remove('drop_time')
cols


# cols = ['vendor_id', 'driver_tip', 'mta_tax', 'payment_method', 'stored_flag']

for x in cols:
  plot_scatter(df[x][:100000],df['total_amount'][:100000],x+" vs. total_amount",x,"total_amount")

# plot_scatter(df['vendor_id'][:100000],df['total_amount'][:100000],"driver_tip vs. total_amount","","total_amount")
# plot_scatter(df['vendor_id'][:100000],df['total_amount'][:100000],"driver_tip vs. total_amount","driver_tip","total_amount")
# plot_scatter(df[''][:100000],df['total_amount'][:100000],"driver_tip vs. total_amount","driver_tip","total_amount")
df['travel_time'] = [ z.total_seconds() for z in (df['drop_time'] - df['pickup_time'])]
df.head()
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
y = df['total_amount']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)



print("Train dataset size: ", X_train.shape[0])
print("Test size: ", X_test.shape[0])


from sklearn.linear_model import LinearRegression

lm = LinearRegression().fit(X_train,y_train)
model_results(X_train, y_train, X_test, y_test, lm)
df['pickup_time'] = pd.to_datetime(df['pickup_time'])
df.dtypes
df['day_of_week'] = df['pickup_time'].dt.dayofweek
df['hour'] = df['pickup_time'].dt.hour
# plot_
df.head()
plot_scatter(df['day_of_week'],df['total_amount'],"day_of_week vs. total_amount","day_of_week","total_amount")
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
y = df['total_amount']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)



print("Train dataset size: ", X_train.shape[0])
print("Test size: ", X_test.shape[0])


from sklearn.linear_model import LinearRegression

lm = LinearRegression().fit(X_train,y_train)
model_results(X_train, y_train, X_test, y_test, lm)<jupyter_output>Train dataset size:  838270
Test size:  209568
----Training Data results----
RMSE: $4.4
R2: 0.91

----Test Data results----
RMSE: $4.3
R2: 0.91<jupyter_text>New Section<jupyter_code>from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler()
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
y = df['total_amount']

sc.fit(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.ensemble import RandomForestRegressor


def rfr_model(X, y):# Perform Grid-Search
  gsc = GridSearchCV(estimator=RandomForestRegressor(),
        param_grid={
          'max_depth': range(10,21),
          'n_estimators': (10, 20, 50, 100),
        },
  cv=5, scoring='neg_mean_squared_error', verbose=0,n_jobs=-1)
  grid_result = gsc.fit(X, y)
  best_params = grid_result.best_params_
  print(best_params)
  rfr = RandomForestRegressor(max_depth=best_params["max_depth"], n_estimators=best_params["n_estimators"],random_state=False, verbose=False)# Perform K-Fold CV
  scores = cross_val_score(rfr, X, y, cv=10, scoring='neg_mean_absolute_error')
  return scores
rfr_model(X,y)
from sklearn.ensemble import RandomForestRegressor

rfr = RandomForestRegressor(n_estimators = 100, max_depth = 40, n_jobs = -1).fit(X_train,y_train)

model_results(X_train, y_train, X_test, y_test, rfr)
rurl='https://raw.githubusercontent.com/roopbiswas/chh-ola/master/test.csv?token=<KEY>'
res=pd.read_csv(rurl,encoding='utf7')
res['stored_flag'] = res['stored_flag'].apply(lambda x: 0 if x=='N' else 1)

res['pickup_time'] = pd.to_datetime(res['pickup_time'])

res['day_of_week'] = res['pickup_time'].dt.dayofweek
res['hour'] = res['pickup_time'].dt.hour
df['vendor_id'] = df['vendor_id'].astype(int)
df['pickup_loc'] = df['pickup_loc'].astype(float)
df['hour'] = df['hour'].astype(float)
res['day_of_week'] = res['day_of_week'].astype(int)
res['stored_flag'] = res['stored_flag'].astype(int)
X = df.drop(['total_amount','ID','pickup_time','drop_time' ], axis=1)
y = df['total_amount']
from sklearn.ensemble import RandomForestRegressor

rfr = RandomForestRegressor(n_estimators = 10, max_depth = 20, n_jobs = -1).fit(X,y)
X_test = res.drop([ 'ID','pickup_time','drop_time' ], axis=1)
X_test.info()
X.info()

y_test_pred = rfr.predict(X_test)
y_test_pred
sub = pd.DataFrame(columns=['ID','total_amount'])
sub['ID'] = res['ID']
sub['total_amount'] = y_test_pred
sub.info()
from google.colab import drive
drive.mount('/content/gdrive')

cd gdrive
cd My Drive
sub.to_csv('submission_chhola.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17298153034300792, 'fraction_numerical': 0.02511873350923483, 'mean_word_length': 7.030508474576271, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '167309', 'license': 'MIT', 'n_tokens_mistral': 3840, 'n_tokens_neox': 3635, 'n_words': 667, 'path': '.ipynb_checkpoints/chhola-checkpoint.ipynb', 'repo_name': 'roopbiswas/chh-ola', 'size': '9523'}