<jupyter_start><jupyter_text>Machine LearningIn this file, instructions how to approach the challenge can be found. We are going to work on different types of Machine Learning problems:- **Regression Problem**: The goal is to predict delay of flights.- **(Stretch) Multiclass Classification**: If the plane was delayed, we will predict what type of delay it is (will be).- **(Stretch) Binary Classification**: The goal is to predict if the flight will be cancelled.<jupyter_code># import pandas
import pandas as pd
import numpy as np
import copy

import seaborn as sns
import matplotlib.pyplot as plt

from scipy import stats
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
from sklearn.decomposition import PCA

from sklearn.model_selection import train_test_split, GridSearchCV, cross_validate, cross_val_score
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor, XGBClassifier, plot_importance
from sklearn.metrics import r2_score, mean_squared_error

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)<jupyter_output>C:\Users\cfsui\anaconda3\lib\site-packages\xgboost\compat.py:36: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
  from pandas import MultiIndex, Int64Index<jupyter_text>Read Preprocessed Data<jupyter_code># load data
df = pd.read_csv("data/flights_preprocessed_62.csv", index_col=0)
df.head(3)<jupyter_output><empty_output><jupyter_text>More Feature Engineering  Transform some new features by using 'arr_delay'<jupyter_code># Split traing and test set first
# In training set, I'm going to add more feature generated by combining 'arr_delay' and some categorial features
# For test set, same features as above mentioned should be added with values computed from training set
# i.e. DON NOT touch target variable in test set from now on
df_train, df_test = train_test_split(df, test_size=0.3)
# calculate how many times has delay ('arr_delay' > 15) happened on each carrier/flight_num/tail_num/carrier/origin_airport/dest_airport/origin_city/origin_state/dest_city/dest_state 
# calculate average/median delay time of each ... (same as above)
# merge with df

tran_features = ['op_unique_carrier', 'tail_num',  'op_carrier_fl_num', 'origin_airport_id', 'dest_airport_id', 'origin_city', 'origin_state', 'dest_city', 'dest_state']

for col in tran_features:
    # delay count group by col
    feature_delay_ct = df_train[df_train['arr_delay'] > 15][[col, 'arr_delay']].groupby(col, as_index=False).count().rename(columns={'arr_delay': f'{col}_delayct'})
    df_train = pd.merge(df_train, feature_delay_ct, on=col, how='left').fillna(0)
    # average delay time group by col
    feature_delay_avg = df[[col, 'arr_delay']].groupby(col, as_index=False).mean().rename(columns={'arr_delay': f'{col}_delayavg'})
    df_train = pd.merge(df_train, feature_delay_avg, on=col, how='left').fillna(0)
    # median delay time group by col
    feature_delay_median = df[[col, 'arr_delay']].groupby(col, as_index=False).median().rename(columns={'arr_delay': f'{col}_delaymedian'})
    df_train = pd.merge(df_train, feature_delay_median, on=col, how='left').fillna(0)
df_train.head()
# reset dtypes
categorical_features = ['op_unique_carrier',
                        'tail_num', 
                        'op_carrier_fl_num',
                        'origin_airport_id',
                        'dest_airport_id',
                        # 'share_code',
                        'origin_city',
                        'origin_state',
                        'dest_city',
                        'dest_state',
                        'fl_month',
                        'fl_weekday',
                        'season',
                        'inbound_fl']

df_train[categorical_features] = df_train[categorical_features].astype('str')
df_test[categorical_features] =df_test[categorical_features].astype('str')
df_test.shape
# add features to test set with values computed by training set
# NOTICE: THE VALUES ADDED HERE ARE STILL FROM TRAINING SET
# for example, flight No.#### used to have 7 delays in training set, then add 7 to same flight No. in test set
# It's like assigning weight to categories (assign weight of 7 to flight No.##### in this example)

origin_new = [['op_unique_carrier', 'op_unique_carrier_delayct', 'op_unique_carrier_delaymedian', 'op_unique_carrier_delayavg'],
              ['tail_num', 'tail_num_delayct', 'tail_num_delaymedian', 'tail_num_delayavg'],
              ['op_carrier_fl_num', 'op_carrier_fl_num_delayct', 'op_carrier_fl_num_delaymedian', 'op_carrier_fl_num_delayavg'],
              ['origin_airport_id', 'origin_airport_id_delayct', 'origin_airport_id_delaymedian', 'origin_airport_id_delayavg'],
              ['dest_airport_id', 'dest_airport_id_delayct', 'dest_airport_id_delaymedian', 'dest_airport_id_delayavg'],
              ['origin_city', 'origin_city_delayct', 'origin_city_delaymedian', 'origin_city_delayavg'],
              ['origin_state', 'origin_state_delayct', 'origin_state_delaymedian', 'origin_state_delayavg'],
              ['dest_city', 'dest_city_delayct', 'dest_city_delaymedian', 'dest_city_delayavg'],
              ['dest_state', 'dest_state_delayct', 'dest_state_delaymedian', 'dest_state_delayavg']]
              
for cols in origin_new:
    df_test = pd.merge(df_test, df_train[cols].drop_duplicates(), on=cols[0], how='left').fillna(0)
df_test.shape<jupyter_output><empty_output><jupyter_text>Transform Target Variable<jupyter_code># target variable distribution
# left skewed with long tail
target_train = df_train['arr_delay']
target_test = df_test['arr_delay']
sns.histplot(data=target_train)
plt.show()
# target variable distribution after log transformation in training set
# If use transformed target variable to train, when evaluate prediction results, REMEMBER to transform predicted value back first
diff = df['arr_delay'].min() - 10 # min value of whole dataset
target_train_log = np.log(target_train - diff)
sns.histplot(data=target_train_log)
plt.show()<jupyter_output><empty_output><jupyter_text>Main Task: Regression Problem The target variable is **ARR_DELAY**. We need to be careful which columns to use and which don't. For example, DEP_DELAY is going to be the perfect predictor, but we can't use it because in real-life scenario, we want to predict the delay before the flight takes of --> We can use average delay from earlier days but not the one from the actual flight we predict.  For example, variables **CARRIER_DELAY, WEATHER_DELAY, NAS_DELAY, SECURITY_DELAY, LATE_AIRCRAFT_DELAY** shouldn't be used directly as predictors as well. However, we can create various transformations from earlier values.We will be evaluating your models by predicting the ARR_DELAY for all flights **1 week in advance**.  Feature Selection / Dimensionality Reduction We need to apply different selection techniques to find out which one will be the best for our problems.- Original Features vs. PCA conponents?  Modeling Use different ML techniques to predict each problem.- linear / logistic / multinomial logistic regression- Naive Bayes- Random Forest- SVM- XGBoost- The ensemble of your own choice<jupyter_code># scores = []
# for f in features:
#     X = pd.get_dummies(data=data[['dep_min_sin', 'dep_min_cos', 'fl_month', 'op_unique_carrier', 'inbound_fl_num', 'arr_min_of_day', 'crs_elapsed_time', 'distance', 'fl_weekday', f]], drop_first=True)
#     y = y_log
    
#     X_scaled = scaler.fit_transform(X)
#     y_scaled = scaler.fit_transform(y)

#     X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_scaled, train_size=0.5, test_size=0.3, random_state=88)
    
#     rr = cross_validate(estimator=Ridge(alpha=0.1),
#                         X=X_train,
#                         y=y_train,
#                         cv=5,
#                         scoring='r2',
#                         return_train_score=True)
    
#     scores.append([f, rr['test_score'].mean()])
# score = pd.DataFrame(scores)
# score[score[1]==score[1].max()]
# X = pd.get_dummies(data=data[['dep_min_sin', 'dep_min_cos', 'fl_month', 'op_unique_carrier', 'inbound_fl_num', 'arr_min_of_day', 'crs_elapsed_time', 'distance', 'fl_wkday_sin', 'fl_wkday_cos']], drop_first=True)
# y = y_log
    
# X_scaled = scaler.fit_transform(X)
# y_scaled = scaler.fit_transform(y)

# X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_scaled, train_size=0.5, test_size=0.3, random_state=88)
    
# rr = cross_validate(estimator=Ridge(alpha=0.1),
#                         X=X_train,
#                         y=y_train,
#                         cv=5,
#                         scoring='r2',
#                         return_train_score=True)
    
# rr['test_score']
# rr['test_score'].mean()<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>avail_features = [
    # 'fl_date',
    # 'op_unique_carrier',
    # 'tail_num', 
    # 'op_carrier_fl_num',
    # 'origin_airport_id',
    # 'dest_airport_id',
    # 'crs_dep_time',
    # 'crs_arr_time',
    # 'crs_elapsed_time',
    'distance',
    # 'share_code',
    # 'origin_city',
    # 'origin_state',
    # 'dest_city',
    # 'dest_state',
    # 'arr_date',
    # 'dep_datetime',
    # 'arr_datetime',
    # 'fl_month',
    # 'fl_weekday',
    # 'season',
    # 'day_num_of_flights',
    'num_flights_6hr',
    'inbound_fl_num',
    # 'inbound_fl',
    # 'dep_min_of_day',
    # 'arr_min_of_day',
    # 'dep_hr',
    # 'arr_hr',
    'arr_min_sin',
    'arr_min_cos',
    # 'arr_hr_sin',
    # 'arr_hr_cos',
    'dep_min_sin', 
    'dep_min_cos', 
    # 'dep_hr_sin', 
    # 'dep_hr_cos',
    'fl_mnth_sin', 
    'fl_mnth_cos',
    'fl_wkday_sin',
    'fl_wkday_cos',
    'op_unique_carrier_delayct',
    'op_unique_carrier_delaymedian',
    # 'op_unique_carrier_delayavg',
    # 'tail_num_delayct', 
    # 'tail_num_delaymedian',
    'tail_num_delayavg',
    # 'op_carrier_fl_num_delayct',
    # 'op_carrier_fl_num_delaymedian', 
    'op_carrier_fl_num_delayavg',
    'origin_airport_id_delayct', 
    # 'origin_airport_id_delaymedian',
    'origin_airport_id_delayavg',
    'dest_airport_id_delayct',
    # 'dest_airport_id_delaymedian',
    'dest_airport_id_delayavg',
    'origin_city_delayct',
    'origin_city_delaymedian', 
    # 'origin_city_delayavg',
    'origin_state_delayct', 
    'origin_state_delaymedian',
    # 'origin_state_delayavg',
    'dest_city_delayct', 
    'dest_city_delaymedian',
    # 'dest_city_delayavg',
    # 'dest_state_delayct',
    # 'dest_state_delaymedian',
    'dest_state_delayavg'
]
X_train = df_train[avail_features]
# y_train = target_train_log
y_train = target_train
X_test = df_test[avail_features]
y_test = target_test

xg_reg = XGBRegressor(objective ='reg:squarederror',
                      learning_rate = 0.1,
                      max_depth = 6,
                      # reg_lambda = 10,
                      n_estimators = 300)
xg_reg.fit(X_train, y_train)
y_pred = xg_reg.predict(X_test)
# y_pred = np.exp(xg_reg.predict(X_test)) + diff
xg_reg.score(X_train, y_train)
xg_reg.score(X_test, y_test)
## Best Score got so far
r2_score(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>pca_features = [
    # 'op_unique_carrier',
    # 'tail_num'.
    # 'op_carrier_fl_num',
    # 'origin_airport_id',
    # 'dest_airport_id',
    'crs_elapsed_time',
    'distance',
    'share_code',
    # 'origin_city',
    # 'origin_state',
    # 'dest_city',
    # 'dest_state',
    'fl_month',
    'fl_weekday',
    'season',
    'day_num_of_flights',
    'num_flights_6hr',
    'inbound_fl_num',
    'inbound_fl',
    'dep_min_of_day',
    'arr_min_of_day',
    'dep_hr',
    'arr_hr',
    'arr_hr_sin',
    'arr_hr_cos',
    'arr_min_sin',
    'arr_min_cos',
    'dep_min_sin',
    'dep_min_cos',
    'dep_hr_sin',
    'dep_hr_cos',
    'fl_mnth_sin',
    'fl_mnth_cos',
    'fl_wkday_sin',
    'fl_wkday_cos',
    'op_unique_carrier_delayct',
    'op_unique_carrier_delaymedian',
    'tail_num_delayct',
    'tail_num_delaymedian',
    'op_carrier_fl_num_delayct',
    'op_carrier_fl_num_delaymedian',
    'origin_airport_id_delayct',
    'origin_airport_id_delaymedian',
    'dest_airport_id_delayct',
    'dest_airport_id_delaymedian',
    'origin_city_delayct',
    'origin_city_delaymedian',
    'origin_state_delayct',
    'origin_state_delaymedian',
    'dest_city_delayct',
    'dest_city_delaymedian',
    'dest_state_delayct',
    'dest_state_delaymedian'
]
df_X = pd.concat([df_train[pca_features], df_test[pca_features]])
df_train.shape[0]
X_scaled = scaler.fit_transform(df_X)

pca = PCA(n_components='mle')
pca.fit(X_scaled)
X_pca = pca.transform(X_scaled)
X_scaled_train = X_pca[:10609, :]
X_scaled_test = X_pca[10609:, :]
y_train = target_train_log
y_test = target_test

xg_reg = XGBRegressor(objective ='reg:squarederror',
                      learning_rate = 0.1,
                      max_depth = 6,
                      # reg_lambda = 10,
                      n_estimators = 300)
xg_reg.fit(X_scaled_train, y_train)
# y_pred = xg_reg.predict(X_test)
y_pred = np.exp(xg_reg.predict(X_scaled_test)) + diff
r2_score(y_test, y_pred)
# features = [
#     # 'op_unique_carrier',
#     # 'tail_num'.
#     # 'op_carrier_fl_num',
#     # 'origin_airport_id',
#     # 'dest_airport_id',
#     # 'crs_elapsed_time',
#     'distance',
#     'share_code',
#     # 'origin_city',
#     # 'origin_state',
#     # 'dest_city',
#     # 'dest_state',
#     # 'fl_month',
#     # 'fl_weekday',
#     # 'season',
#     # 'day_num_of_flights',
#     # 'num_flights_6hr',
#     # 'inbound_fl_num',
#     # 'inbound_fl',
#     # 'dep_min_of_day',
#     # 'arr_min_of_day',
#     # 'dep_hr',
#     # 'arr_hr',
#     # 'arr_hr_sin',
#     # 'arr_hr_cos',
#     # 'arr_min_sin',
#     # 'arr_min_cos',
#     'dep_min_sin',
#     # 'dep_min_cos',
#     # 'dep_hr_sin',
#     # 'dep_hr_cos',
#     # 'fl_mnth_sin',
#     # 'fl_mnth_cos',
#     # 'fl_wkday_sin',
#     # 'fl_wkday_cos',
#     # 'op_unique_carrier_delayct',
#     # 'op_unique_carrier_delaymedian',
#     'tail_num_delayct',
#     # 'tail_num_delaymedian',
#     'op_carrier_fl_num_delayct',
#     # 'op_carrier_fl_num_delaymedian',
#     # 'origin_airport_id_delayct',
#     # 'origin_airport_id_delaymedian',
#     # 'dest_airport_id_delayct',
#     # 'dest_airport_id_delaymedian',
#     # 'origin_city_delayct',
#     'origin_city_delaymedian',
#     # 'origin_state_delayct',
#     'origin_state_delaymedian',
#     'dest_city_delayct',
#     # 'dest_city_delaymedian',
#     # 'dest_state_delayct',
#     'dest_state_delaymedian'
# ]
# scores = []
# for f in features:
#     X_train = df_train[[f]]
#     y_train = target_train_log
#     X_test = df_test[[f]]
#     y_test = target_test
    
#     xg_reg = XGBRegressor(objective ='reg:squarederror',
#                       learning_rate = 0.1,
#                       max_depth = 6,
#                       # reg_lambda = 10,
#                       n_estimators = 300)
#     xg_reg.fit(X_train, y_train)
#     y_pred = np.exp(xg_reg.predict(X_test)) + diff
#     # y_pred = xg_reg.predict(X_test)
    
#     scores.append([f, xg_reg.score(X_train, y_train), r2_score(y_test, y_pred)])
# s = pd.DataFrame(scores)
# s[s[2]==s[2].max()]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11918735891647855, 'fraction_numerical': 0.006062560464366333, 'mean_word_length': 3.657855211775308, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '469945', 'license': 'MIT', 'n_tokens_mistral': 5671, 'n_tokens_neox': 5437, 'n_words': 1044, 'path': 'experiments/modeling_weight_random.ipynb', 'repo_name': 'DDDDDDaisyS/LHL_Midterm_Project_I', 'size': '15505'}