<jupyter_start><jupyter_text>Zindi UmojaHack Africa 2021 2: Sendy - Delivery Rider Response Challenge (INTERMEDIATE) by UmojaHack Africa This notebook was created in Google Colab<jupyter_code># Importing Libraries
import requests
from pydrive.auth import GoogleAuth 
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Set user authentification path
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# logging to avoid discovery_cache error
import logging
logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)

# Getting Id of data file and load data in Google Colab
# https://drive.google.com/drive/folders/16RYdhdDcj_glWKjXR7OnT4bfcSXUymYw?usp=sharing
file_list = drive.ListFile({'q': "'16RYdhdDcj_glWKjXR7OnT4bfcSXUymYw' in parents and trashed=false"}).GetList()

# View files in the Drive Folder
for file1 in file_list:
  print('title: %s, id: %s' % (file1['title'], file1['id']))
# Loading the Data from the source i.e. csv
import os
from io import StringIO

# Now, we will be getting content of file by using id.
# Train file
train_file =  drive.CreateFile({'id': '1V_4PpDw_HyxlJldsLVp_jD4MKJL3Z3TF'})
train_file.GetContentFile('Train.csv')
# Test file
test_file =  drive.CreateFile({'id': '1IvFdJrFh5D7BCLdxIgkdc3TWDjmTMaOQ'})
test_file.GetContentFile('Test.csv')
# Rider file
rider_file =  drive.CreateFile({'id': '1i8rR25xDMe73IxqGdj4CKL64yWxj4ypP'})
rider_file.GetContentFile('Riders.csv')
# Data URL's
train_url = 'https://drive.google.com/file/d/1V_4PpDw_HyxlJldsLVp_jD4MKJL3Z3TF/view?usp=sharing'
test_url = 'https://drive.google.com/file/d/1IvFdJrFh5D7BCLdxIgkdc3TWDjmTMaOQ/view?usp=sharing'
rider_url = 'https://drive.google.com/file/d/1i8rR25xDMe73IxqGdj4CKL64yWxj4ypP/view?usp=sharing'

# Function to read CSV file
def read_csv(url):
  url = 'https://drive.google.com/uc?export=download&id=' + url.split('/')[-2]
  csv_raw = requests.get(url).text
  csv = StringIO(csv_raw)
  return csv

# Load the data
import pandas as pd
df_train = pd.read_csv(read_csv(train_url))
df_test = pd.read_csv(read_csv(test_url))
df_riders = pd.read_csv(read_csv(rider_url))
# Creating Copy to work with
train = df_train.copy()
test = df_test.copy()
riders = df_riders.copy()
# Determining the no. of records in our datasets
print('The train dataset has ', train.shape[1], ' columns and ', train.shape[0], ' rows!')
print('The test dataset has ', test.shape[1], ' columns and ', test.shape[0], ' rows!')
print('The test dataset has ', riders.shape[1], ' columns and ', riders.shape[0], ' rows!')
# Separate target variable 
target = train['target']
# Drop target column then join Train and test
train.drop('target',axis = 1, inplace = True)

# Merge the train and test dataframe
frames = [train,test]
df = pd.concat(frames)
# Checking unique values and number of rows asigned for each column in the dataset
for i in df.columns:
  
  print('\n')
  
  print(i,df[i].nunique())

  print(i,df[i].unique())<jupyter_output>ID 256658
ID ['ID_SCUW21PVAU' 'ID_2HA7X30JMN' 'ID_IAJWDTBY6M' ... 'ID_VDI8L0SWNS'
 'ID_7L22AF1T6K' 'ID_LMMA33Q7YL']


order_id 64326
order_id [ 4435 32711  8712 ... 11467 12867 12617]


dispatch_day 31
dispatch_day [27 30 14 22  3 18 28 20  6 12 17  7  5  1 15 31 16  4 25 19 11 23 21  9
 13 10 26  2  8 29 24]


dispatch_day_of_week 7
dispatch_day_of_week [6 7 2 3 5 4 1]


dispatch_time 5788
dispatch_time ['09:02:54' '13:01:37' '10:01:00' ... '06:09:58' '20:08:49' '05:09:47']


client_id 5457
client_id [ 593630  837729  695129 ...  676936 2533910 2361571]


client_type 2
client_type ['Business' 'Personal']


order_license_status 2
order_license_status [0 1]


order_carrier_type 3
order_carrier_type [2 1 0]


vendor_type 1
vendor_type ['Bike']


rider_id 775
rider_id [ 30153  20884  33143  96531 103546  66125  66033  45701 117622  40066
  74842  93610   9729 117093  80132  93633 117760 115184  62445 118082
  76406   5865 118588 100441   7383 105892  66010 106398 100855  64561
  53521  [...]<jupyter_text>Feature Engineering  Rider Amount Class<jupyter_code># Let's view the distribution of the rider's payment for the dispatch
hist_amount = df.rider_amount.hist(bins=25,figsize=[15,10])
import numpy as np
# create a list of our conditions
conditions = [
    (df['rider_amount'] <= 390),
    (df['rider_amount'] > 390) & (df['rider_amount'] <= 650),
    (df['rider_amount'] > 650) & (df['rider_amount'] <= 820),
    (df['rider_amount'] > 800) & (df['rider_amount'] <= 1200)
    ]

# create a list of the values we want to assign for each condition
values = ['2', '4', '3', '1']

# create a new column and use np.select to assign values to it using our lists as arguments
df['rider_amount_range'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Day of month<jupyter_code># Dispatches frequency per day of month
df['dispatch_day'].value_counts().plot(kind='bar',figsize=[15,5])
# create a list of our conditions
conditions = [
    (df['dispatch_day'] == 1),
    (df['dispatch_day'] > 1) & (df['dispatch_day'] <= 6),
    (df['dispatch_day'] > 6) & (df['dispatch_day'] <= 8),
    (df['dispatch_day'] >8)
    ]

# create a list of the values we want to assign for each condition
values = ['1', '2','3','4']

# create a new column and use np.select to assign values to it using our lists as arguments
df['dispatch_day_range'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Weekends<jupyter_code># Dispatches frequency per  weekday
df['dispatch_day_of_week'].value_counts().plot(kind='bar',figsize=[15,5])
# create a list of our conditions
conditions = [
    (df['dispatch_day_of_week'] == 1),
    (df['dispatch_day_of_week'] == 7)
    ]

# create a list of the values we want to assign for each condition
values = ['2', '1']

# create a new column and use np.select to assign values to it using our lists as arguments
df['weekend'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Busy Hours<jupyter_code># Convert time column to datetime
df['dispatch_time'] = pd.to_datetime(df['dispatch_time'])

# Extract Hour Column
df['dispatch_hour'] = df['dispatch_time'].dt.hour

# Let's view the hourly distribution of dispatches
df['dispatch_hour'].value_counts().plot(kind='bar',figsize=[15,5])
# Let's get the cos and cosine of our Time Variable
time_column = ["dispatch_time"]

for col in time_column:
  df[col.split('_')[0] + '_hour_sine'] = np.sin(2 * np.pi * df[col].dt.hour/24)
  df[col.split('_')[0] + '_hour_cos'] = np.cos(2* np.pi * df[col].dt.hour/24)
# create a list of our conditions
conditions = [
    (df['dispatch_hour'] <= 6),
    (df['dispatch_hour'] > 8) & (df['dispatch_hour'] <= 16),
    (df['dispatch_hour'] > 16) & (df['dispatch_hour'] <= 20),
    (df['dispatch_hour'] > 20)
    ]

# create a list of the values we want to assign for each condition
values = ['1', '4', '3', '2']

# create a new column and use np.select to assign values to it using our lists as arguments
df['busy_hours'] = np.select(conditions, values)

# display updated DataFrame
df.head(2)<jupyter_output><empty_output><jupyter_text>Geo Distance<jupyter_code># Calculate distance between coordinates

def distance_calc(lat1, lng1, lat2, lng2):
    lat1, lng1, lat2, lng2 = map(np.radians, (lat1, lng1, lat2, lng2))
    
    AVG_EARTH_RADIUS = 6371  # in km
    
    lat = lat2 - lat1
    
    lng = lng2 - lng1
    
    d = np.sin(lat * 0.5) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(lng * 0.5) ** 2
    
    h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))
    
    return h
import numpy as np

# Let's create variables for the distances between the locations of the riders, pickup points and dropoff points
df['rider_dropoff_dist'] = distance_calc(df.rider_lat, df.rider_long, df.drop_off_lat, df.drop_off_long)
df['rider_pickup_dist'] = distance_calc(df.rider_lat, df.rider_long, df.pickup_lat, df.pickup_long)
df['pickup_dropoff_dist'] = distance_calc(df.pickup_lat, df.pickup_long, df.drop_off_lat, df.drop_off_long)<jupyter_output><empty_output><jupyter_text>Distance Ranges Pickup Drop_off ranges<jupyter_code># We can now view the distribution of the distance between pickup and drop off locations

bin_values = np.arange(start=-0, stop=60, step=0.3)
df['pickup_dropoff_dist'].hist(bins=bin_values, figsize=[14,6])
# create a list of our conditions
conditions = [
    (df['pickup_dropoff_dist'] <= 6),
    (df['pickup_dropoff_dist'] > 6) & (df['pickup_dropoff_dist'] <= 16),
    (df['pickup_dropoff_dist'] > 16) & (df['pickup_dropoff_dist'] <= 30),
    (df['pickup_dropoff_dist'] > 30) & (df['pickup_dropoff_dist'] <= 50),
    (df['pickup_dropoff_dist'] > 50)
    ]

# create a list of the values we want to assign for each condition
values = ['5','4', '3', '2', '1']

# create a new column and use np.select to assign values to it using our lists as arguments
df['pickup_dropoff_dist_range'] = np.select(conditions, values)

# display updated DataFrame
df.head(2)<jupyter_output><empty_output><jupyter_text>Rider_Pickup_Ranges<jupyter_code># Let's also look at distribution of the distance between the rider and the pickup location

bin_values = np.arange(start=-0, stop=4, step=0.1)
df['rider_pickup_dist'].hist(bins=bin_values, figsize=[14,6])
# create a list of our conditions
conditions = [
    (df['rider_pickup_dist'] <= 1),
    (df['rider_pickup_dist'] > 1) & (df['rider_pickup_dist'] <= 1.5),
    (df['rider_pickup_dist'] > 1.5) & (df['rider_pickup_dist'] <= 2.5),
    (df['rider_pickup_dist'] > 2.5) & (df['rider_pickup_dist'] <= 3.7),
    (df['rider_pickup_dist'] > 3.7)
    ]

# create a list of the values we want to assign for each condition
values = ['1', '2', '3', '4','5']

# create a new column and use np.select to assign values to it using our lists as arguments
df['rider_pickup_dist_range'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Rider Drop_off_Ranges<jupyter_code># Let's also visualize the distance between the rider and drop off location

bin_values = np.arange(start=-0, stop=30, step=0.3)
df['rider_dropoff_dist'].hist(bins=bin_values, figsize=[14,6])
# create a list of our conditions
conditions = [
    (df['rider_dropoff_dist'] <= 5),
    (df['rider_dropoff_dist'] > 5) & (df['rider_dropoff_dist'] <= 10),
    (df['rider_dropoff_dist'] > 10) & (df['rider_dropoff_dist'] <= 18),
    (df['rider_dropoff_dist'] > 18) & (df['rider_dropoff_dist'] <= 30),
    (df['rider_dropoff_dist'] > 30)
    ]

# create a list of the values we want to assign for each condition
values = ['1', '2', '3', '4','5']

# create a new column and use np.select to assign values to it using our lists as arguments
df['rider_dropoff_dist_range'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Time and Distance<jupyter_code># create a list of our conditions
conditions = [
    (df['rider_dropoff_dist_range'] == '5') & (df['busy_hours'] == '1'),
    (df['rider_dropoff_dist_range'] == '4') & (df['busy_hours'] == '2'),
    (df['rider_dropoff_dist_range'] == '3') & (df['busy_hours'] == '3'),
    (df['rider_dropoff_dist_range'] == '2') & (df['busy_hours'] == '4'),
    (df['rider_dropoff_dist_range'] == '1') & (df['busy_hours'] == '4'),
    (df['rider_dropoff_dist_range'] == '2') & (df['busy_hours'] == '0')
    ]

# create a list of the values we want to assign for each condition
values = ['1', '2', '3', '4','6','5']

# create a new column and use np.select to assign values to it using our lists as arguments
df['time_distance_bias'] = np.select(conditions, values)

# display updated DataFrame
df.head(2)<jupyter_output><empty_output><jupyter_text>Weekday vs Time_distance<jupyter_code># create a list of our conditions
conditions = [
    (df['dispatch_day_of_week'] == '1') & (df['time_distance_bias'] == '5'), # Sunday  early morning & average distance
    (df['dispatch_day_of_week'] == '7') & (df['time_distance_bias'] == '3'), # Saturday late night & long distance
    (df['dispatch_day_of_week'] == '7') & (df['time_distance_bias'] == '5'), # Saturday early morning average distance
    (df['dispatch_day_of_week'] == '0') & (df['time_distance_bias'] == '3'), # Weekday late night & long distance
    (df['dispatch_day_of_week'] == '0') & (df['time_distance_bias'] == '6')  # Weekday daytime and average distance
    ]

# create a list of the values we want to assign for each condition
values = ['1', '2', '3', '4','5']

# create a new column and use np.select to assign values to it using our lists as arguments
df['weekend_time_distance_bias'] = np.select(conditions, values)

# display updated DataFrame
df.head(2)<jupyter_output><empty_output><jupyter_text>Money Earned VS time & Distance<jupyter_code># create a list of our conditions
conditions = [
    (df['rider_amount_range'] == '4') & (df['busy_hours'] == '4'),
    (df['rider_pickup_dist_range'] == '3') & (df['busy_hours'] == '2'),
    (df['rider_pickup_dist_range'] == '1') & (df['rider_amount_range'] == '4'),
    (df['rider_pickup_dist_range'] == '4') & (df['rider_amount_range'] == '4')
    ]

# create a list of the values we want to assign for each condition
values = ['4','2','6','1']

# create a new column and use np.select to assign values to it using our lists as arguments
df['money_distance_bias'] = np.select(conditions, values)

# display updated DataFrame
df.head(3)<jupyter_output><empty_output><jupyter_text>Correcting Dtypes<jupyter_code># let's convert some string columns to numeric
df[['rider_dropoff_dist_range',	'time_distance_bias',	'weekend_time_distance_bias','busy_hours','weekend','money_distance_bias']] = df[['rider_dropoff_dist_range',	'time_distance_bias',	'weekend_time_distance_bias','busy_hours','weekend','money_distance_bias']].apply(pd.to_numeric)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># Lets install catboost for creating a model
!pip install catboost
# Split train and test set
train = df.iloc[:179867, :]
test = df.iloc[179867:, :]
# Selecting the columns to be used as data for accuracy testing
cols = [col for col in train.columns if col not in ['dispatch_time']]

# Set the target and independent variables
X = train[cols]
y = target
test.drop(['dispatch_time'],axis = 1, inplace=True)
# Fitting the data to model
from catboost import CatBoostClassifier

# You can increase the iterations parameter but it will take longer to learn
model = CatBoostClassifier(iterations = 1000, early_stopping_rounds = 50)

# Here we fit our data then make predictions
import numpy as np

# Since Catboost accepts categorical features we'll make then identifiable to the model 
categorical_features_indices = np.where(X.dtypes != np.float)[0]

# Fitting our data to the model
model.fit(X,y,cat_features=categorical_features_indices)

# Making Predictions
y_pred = model.predict(test)
model.score(X,y)
# Let's create a submission file to input our predictions
submissionFile= pd.DataFrame({"ID":df_test["ID"]})
submissionFile['target'] = y_pred

# Saving our data in a CSV
submissionFile.to_csv("submit.csv",index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13836313797154418, 'fraction_numerical': 0.045685941783057044, 'mean_word_length': 5.337055417700579, 'pattern_counts': {'":': 1, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '510305', 'license': 'Apache-2.0', 'n_tokens_mistral': 6015, 'n_tokens_neox': 5511, 'n_words': 1450, 'path': 'UmojaHack_2021_Sendy_002.ipynb', 'repo_name': 'Mburia/2nd-Place-Solution-UmojaHack_2021-Sendy', 'size': '15322'}