<jupyter_start><jupyter_text>Yassir ETA prediction using CatBoost  I - Import libraries - pandas (https://pandas.pydata.org/docs/)- numpy (https://numpy.org/doc/stable/)- scikit-learn (https://scikit-learn.org/stable/)- catboost (https://catboost.ai/docs/concepts/about.html)- matplotlib (https://matplotlib.org/contents.html)<jupyter_code>import pandas
import numpy
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.cluster import MiniBatchKMeans
from catboost import CatBoostRegressor
from matplotlib import pyplot
%matplotlib inline<jupyter_output><empty_output><jupyter_text>II - Load data<jupyter_code>train = pandas.read_csv('Train.csv')
test = pandas.read_csv('Test.csv')<jupyter_output><empty_output><jupyter_text>III - Initial data exploration  1 - File structure<jupyter_code>train.head()<jupyter_output><empty_output><jupyter_text>2 - Data summary<jupyter_code># train, test, #train_col, # test_col
train.shape[0], test.shape[0], train.shape[1], test.shape[1]
train.describe()<jupyter_output><empty_output><jupyter_text>IV - Data preparation<jupyter_code># converting time
import datetime
train['start_time'] = pandas.to_datetime(train['Timestamp'])
test['start_time'] = pandas.to_datetime(test['Timestamp'])
# checking for missing values 
train.count().min() == train.shape[0] and test.count().min() == test.shape[0]<jupyter_output><empty_output><jupyter_text>1 - checking the distribution of the trip duration<jupyter_code>train['Trip_duration_hr'] = train['ETA']/3600
train['Trip_duration_hr'] = train['Trip_duration_hr'].values
pyplot.hist(train['Trip_duration_hr'].values, bins=100)
pyplot.xlabel('Trip_duration_hr')
pyplot.ylabel('records')
pyplot.show()
train['log_duration'] = numpy.log1p(train['ETA'].values)
pyplot.hist(train['log_duration'].values, bins=100)
pyplot.xlabel('log_duration')
pyplot.ylabel('records')
pyplot.show()<jupyter_output><empty_output><jupyter_text>2 - Applying log transformation to reduce the skeweness<jupyter_code>pyplot.scatter(range(train.shape[0]), numpy.sort(train.log_duration.values))
pyplot.xlabel('index')
pyplot.ylabel('log_duration')
pyplot.show()
print(len(train[train['log_duration'] < 1.5]))
train = train[train['log_duration'] > 1.5]<jupyter_output>104<jupyter_text>3 - checking if the period of the train and test datasets overlap<jupyter_code>train.loc[:, 'start_date'] = train['start_time'].dt.date
test.loc[:, 'start_date'] = test['start_time'].dt.date
pyplot.plot(train.groupby('start_date').count()[['ID']], 'o-', label='train')
pyplot.plot(test.groupby('start_date').count()[['ID']], 'o-', label='test')
pyplot.title('Train and test periods overlap ?')
pyplot.legend(loc=0)
pyplot.ylabel('number of records')
pyplot.show()<jupyter_output><empty_output><jupyter_text>V - Feature engineering<jupyter_code>train['hour_pick']=train.start_time.dt.hour
train['day_of_week']=train.start_time.dt.dayofweek
train['is_night_time']=[1 if (i==0 or i>=19)  else 0 for i in train['start_time'].dt.hour]
train['late_night_time']=[1 if (i<5 or i>0)  else 0 for i in train['start_time'].dt.hour]
train['min_of_pick']=train['start_time'].dt.minute
train['center_lat']=(train['Destination_lat']-train['Origin_lat'])/2
train['center_long']=(train['Destination_lon']-train['Origin_lon'])/2

test['hour_pick']=test.start_time.dt.hour
test['day_of_week']=test.start_time.dt.dayofweek
test['is_night_time']=[1 if (i==0 or i>=19)  else 0 for i in test['start_time'].dt.hour]
test['late_night_time']=[1 if (i<5 or i>0)  else 0 for i in test['start_time'].dt.hour]
test['min_of_pick']=train['start_time'].dt.minute
test['center_lat']=(test['Destination_lat']-test['Origin_lat'])/2
test['center_long']=(test['Destination_lon']-test['Origin_lon'])/2

# difference between pickup and dropoff latitudes
train['lat_diff'] = train['Origin_lat'] - train['Destination_lat']
train['lon_diff'] = train['Origin_lon'] - train['Destination_lon']

# difference between pickup and dropoff longitudes
test['lat_diff'] = test['Origin_lat'] - test['Destination_lat']
test['lon_diff'] = test['Origin_lon'] - test['Destination_lon']
coords = numpy.vstack((train[['Origin_lat', 'Origin_lon', 'Trip_distance']].values,
                    train[['Destination_lat', 'Destination_lon', 'Trip_distance']].values))
numpy.random.seed(1234)
sample_ind = numpy.random.permutation(len(coords))
kmeans = MiniBatchKMeans(n_clusters=90, batch_size=10000, random_state=1234).fit(coords[sample_ind])
train.loc[:, 'start_cluster'] = kmeans.predict(train[['Origin_lat', 'Origin_lon', 'Trip_distance']])
train.loc[:, 'end_cluster'] = kmeans.predict(train[['Destination_lat', 'Destination_lon', 'Trip_distance']])
test.loc[:, 'start_cluster'] = kmeans.predict(test[['Origin_lat', 'Origin_lon', 'Trip_distance']])
test.loc[:, 'end_cluster'] = kmeans.predict(test[['Destination_lat', 'Destination_lon', 'Trip_distance']])
coords = numpy.vstack((train[['Origin_lat', 'Origin_lon', 'Trip_distance']].values,
                    train[['Destination_lat', 'Destination_lon', 'Trip_distance']].values,
                    test[['Origin_lat', 'Origin_lon', 'Trip_distance']].values,
                    test[['Destination_lat', 'Destination_lon', 'Trip_distance']].values))

pca = PCA().fit(coords)
train['start_pca0'] = pca.transform(train[['Origin_lat', 'Origin_lon', 'Trip_distance']])[:, 0]
train['start_pca1'] = pca.transform(train[['Origin_lat', 'Origin_lon', 'Trip_distance']])[:, 1]
train['end_pca0'] = pca.transform(train[['Destination_lat', 'Destination_lon', 'Trip_distance']])[:, 0]
train['end_pca1'] = pca.transform(train[['Destination_lat', 'Destination_lon', 'Trip_distance']])[:, 1]
test['start_pca0'] = pca.transform(test[['Origin_lat', 'Origin_lon', 'Trip_distance']])[:, 0]
test['start_pca1'] = pca.transform(test[['Origin_lat', 'Origin_lon', 'Trip_distance']])[:, 1]
test['end_pca0'] = pca.transform(test[['Destination_lat', 'Destination_lon', 'Trip_distance']])[:, 0]
test['end_pca1'] = pca.transform(test[['Destination_lat', 'Destination_lon', 'Trip_distance']])[:, 1]
cols_to_drop = ['ID', 'Timestamp', 'start_time', 'start_date', 'min_of_pick']
train_id = train['ID'].values
test_id = test['ID'].values

train_y = train.log_duration.values
train_X = train.drop(cols_to_drop + ['ETA', 'log_duration', 'Trip_duration_hr'], axis=1)
test_X = test.drop(cols_to_drop, axis=1)<jupyter_output><empty_output><jupyter_text>VI - Training the model<jupyter_code>categorical_features_indices = numpy.where(train_X.dtypes != numpy.float)[0]
categorical_features_indices
X_train, X_validation, y_train, y_validation = train_test_split(train_X, train_y, train_size=0.95, random_state=55)
model=CatBoostRegressor(iterations=100, learning_rate=0.1 ,loss_function="RMSE",eval_metric="MAE", random_state=55)
model.fit(X_train, y_train,cat_features=categorical_features_indices,eval_set=(X_validation, y_validation),plot=True)<jupyter_output><empty_output><jupyter_text>VII - Predicting the ETA<jupyter_code>pres = model.predict(test_X)
test['log_duration'] = pandas.DataFrame(pres)
test['ETA'] = numpy.expm1(test['log_duration'])
test[['ID', 'ETA']].to_csv('pred.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17797319932998326, 'fraction_numerical': 0.013958682300390842, 'mean_word_length': 8.801641586867305, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '486220', 'license': 'MIT', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2570, 'n_words': 375, 'path': 'Yassir ETA Prediction.ipynb', 'repo_name': 'JustDhia/YASSIR-ETA-prediction', 'size': '7164'}