<jupyter_start><jupyter_text>Scope of this workbookHere, we want to actually model the whole thing for different datasets.<jupyter_code># Imports
import pandas as pd 
import numpy as np 
import seaborn as sns
import matplotlib.pyplot as plt

import datetime

from sklearn.preprocessing import OneHotEncoder
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error


#Settings
pd.set_option("display.max_rows", 70)
pd.set_option("display.max_columns", 101)<jupyter_output><empty_output><jupyter_text>full_df_unedited<jupyter_code>full_df_unedited = pd.read_pickle('../data/processed/full_df_unedited.pkl')
full_df_unedited.dtypes
full_df_unedited.head()
# For the moment, let's in this step drop all variables we will not use. ToDo: Check which step is the best step to do that
columns_to_drop = [
    'fn_number',
    'TLC_trans',
    'catering_duration',
    'dep_sched_date', # the date itself has no real information. ToDo: Maybe we should extract the day of week here
    'dep_sched_time', # the time has a value but is dropped for the moment due to formatting. 
    'arr_sched_date',
    'arr_sched_time',
    'm_offblockdt',
    'm_onblockdt'
]

full_df_unedited = full_df_unedited.drop(columns_to_drop, axis = 1)

full_df_unedited = full_df_unedited[full_df_unedited['Act Groundtime']<180]
# Creating dummy variables for all categorical variables

# Note: Onehotencoder is the better solution, however for simplicity let's use pandas for the moment
#full_df_unedited_encoded = OneHotEncoder().fit_transform(full_df_unedited)
#full_df_unedited

# Get object columns
full_df_unedited_objectcolumns = full_df_unedited.select_dtypes(include = 'object')
varlist = full_df_unedited_objectcolumns.columns.values.tolist()

#get dummies
full_df_unedited_encoded = pd.get_dummies(full_df_unedited, columns = varlist, drop_first = True)
full_df_unedited_encoded.shape
# We split the forecast in two different forecasts - one for ground time and one for block time

X_train_block, X_test_block, y_train_block, y_test_block = train_test_split(full_df_unedited_encoded.drop(['block_time'], axis = 1), full_df_unedited_encoded['block_time'], test_size=0.33, random_state=42)
X_train_ground, X_test_ground, y_train_ground, y_test_ground = train_test_split(full_df_unedited_encoded.drop(['Act Groundtime'], axis = 1), full_df_unedited_encoded['Act Groundtime'], test_size=0.33, random_state=42)
# Gradientboosting as base classifier

# Blocktime
base_model_block = GradientBoostingRegressor()

base_model_block.fit(X_train_block, y_train_block)
base_model_block.predict(X_test_block)


# Groundtime

base_model_ground = GradientBoostingRegressor()

base_model_ground.fit(X_train_ground, y_train_ground)
base_model_ground.predict(X_test_ground)

# Blocktime
r2_block = base_model_block.score(X_test_block, y_test_block)
rmse_block = np.sqrt(mean_squared_error(y_test_block, base_model_block.predict(X_test_block)))
print("The r^2 for Block time is " + str(round(r2_block,4)))
print("The RMSE for Block time is " + str(round(rmse_block,2)) + " minutes.")
print("\n")

# Groundtime
r2_ground = base_model_ground.score(X_test_ground, y_test_ground)
rmse_ground = np.sqrt(mean_squared_error(y_test_ground, base_model_ground.predict(X_test_ground)))
print("The r^2 for Ground time is " + str(round(r2_ground,4)))
print("The RMSE for Ground time is " + str(round(rmse_ground,2)) + " minutes.")<jupyter_output>The r^2 for Block time is 0.8066
The RMSE for Block time is 18.22 minutes.


The r^2 for Ground time is 0.8003
The RMSE for Ground time is 24.98 minutes.<jupyter_text>full_df_unedited with delay prediction<jupyter_code>full_df = pd.read_pickle('../data/processed/full_df_unedited_wdelay.pkl')
full_df.head()<jupyter_output><empty_output><jupyter_text>We repeat the basic experiment, but this time we do delay prediction instead of ground/block time prediction<jupyter_code># For the moment, let's in this step drop all variables we will not use. ToDo: Check which step is the best step to do that
columns_to_drop_delaypred = [
    'fn_number',
    'TLC_trans',
    'catering_duration',
    'dep_sched_date', # the date itself has no real information. ToDo: Maybe we should extract the day of week here
    'dep_sched_time', # the time has a value but is dropped for the moment due to formatting. 
    'arr_sched_date',
    'arr_sched_time',
    'm_offblockdt',
    'm_onblockdt',
    # to ensure proper comparison in between ground/block time and ground/block delay prediction, we drop the targets of ground/block time prediction. 
    'Act Groundtime',
    #'block_time'
]

full_df = full_df[full_df['Act Groundtime']<180]


full_df = full_df.drop(columns_to_drop_delaypred, axis = 1)
# Creating dummy variables for all categorical variables
# Note: Onehotencoder is the better solution, however for simplicity let's use pandas for the moment

# Get object columns
full_df_objectcolumns = full_df.select_dtypes(include = 'object')
varlist = full_df_objectcolumns.columns.values.tolist()

# get dummies
full_df_encoded = pd.get_dummies(full_df, columns = varlist, drop_first = True)
# We split the forecast in two different forecasts - one for ground delay and one for block delay

X_train_blockdelay, X_test_blockdelay, y_train_blockdelay, y_test_blockdelay = train_test_split(
    full_df_encoded.drop(['block_delay'], axis = 1), full_df_encoded['block_delay'], test_size=0.33, random_state=42)
    
X_train_grounddelay, X_test_grounddelay, y_train_grounddelay, y_test_grounddelay = train_test_split(
    full_df_encoded.drop(['ground_delay'], axis = 1), full_df_encoded['ground_delay'], test_size=0.33, random_state=42)

# Gradientboosting as base classifier

# Blocktime
base_model_blockdelay = GradientBoostingRegressor()

base_model_blockdelay.fit(X_train_blockdelay, y_train_blockdelay)
base_model_blockdelay.predict(X_test_blockdelay)


# Groundtime

base_model_grounddelay = GradientBoostingRegressor()

base_model_grounddelay.fit(X_train_grounddelay, y_train_grounddelay)
base_model_grounddelay.predict(X_test_grounddelay)

# Blocktime
r2_blockdelay = base_model_blockdelay.score(X_test_blockdelay, y_test_blockdelay)
rmse_blockdelay = np.sqrt(mean_squared_error(y_test_blockdelay, base_model_blockdelay.predict(X_test_blockdelay)))
print("The r^2 for Block delay is " + str(round(r2_blockdelay,4)))
print("The RMSE for Block delay is " + str(round(rmse_blockdelay,2)) + " minutes.")
print("\n")

# Groundtime
r2_grounddelay = base_model_grounddelay.score(X_test_grounddelay, y_test_grounddelay)
rmse_grounddelay = np.sqrt(mean_squared_error(y_test_grounddelay, base_model_grounddelay.predict(X_test_grounddelay)))
print("The r^2 for Ground delay is " + str(round(r2_grounddelay,4)))
print("The RMSE for Ground delay is " + str(round(rmse_grounddelay,2)) + " minutes.")<jupyter_output>The r^2 for Block delay is 0.3431
The RMSE for Block delay is 5.63 minutes.


The r^2 for Ground delay is 0.9355
The RMSE for Ground delay is 7.79 minutes.
{'fraction_non_alphanumeric': 0.08664772727272728, 'fraction_numerical': 0.013494318181818182, 'mean_word_length': 6.628385698808234, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '231868', 'license': 'FTL', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2340, 'n_words': 586, 'path': 'notebooks/5modelling.ipynb', 'repo_name': 'Paul1911/Delay-Prediction', 'size': '7040'}