<jupyter_start><jupyter_text>Models on Integrated Features<jupyter_code>import pandas as pd
import numpy as np
import pickle

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import GridSearchCV
# load data
df = pd.read_csv('data/model_df_weather.csv', index_col=0)
df.head(1)<jupyter_output><empty_output><jupyter_text>Model 4: Simple LinReg<jupyter_code>df.columns
x_cols = ['carrier_speed_rank', 'flight_num_speed_rank','month_rank',
       'dep_hour_rank', 'arr_hour_rank', 'precip_cat', 'snow_cat',
          'windgust_cat', 'cloud_cat', 'fl_month',
       'airline_delay', 'haul_length', 'dep_timeday', 'arr_timeday',
       'busy_origin', 'busy_dest']

X = df[x_cols].to_numpy()
y= df.arr_delay.to_numpy()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
lr = LinearRegression()
lr.fit(X_train, y_train)
# filename = 'models/lr4.sav'
# pickle.dump(lr, open(filename, 'wb'))
lr.score(X_test, y_test)
# filename = 'models/lr_0.07.sav'
# pickle.dump(lr, open(filename, 'wb'))<jupyter_output><empty_output><jupyter_text>Model5: <NAME> Random Forest - defaults<jupyter_code>rf1 = RandomForestRegressor()
rf1.fit(X_train, y_train)

# filename = 'models/rf1.sav'
# pickle.dump(rf1, open(filename, 'wb'))
pickle.load( open( "'models/rf1.sav'", "rb" ))
y_pred = rf1.predict(X_test)
print(r2_score(y_test, y_pred))<jupyter_output>0.00611263205182766<jupyter_text>Model 6: Random Forest with 500 estimators<jupyter_code>rf2 = RandomForestRegressor(n_estimators=500)
rf2.fit(X_train, y_train)

filename = 'models/rf2.sav'
pickle.dump(rf2, open(filename, 'wb'))
y_pred = rf2.predict(X_test)
print(r2_score(y_test, y_pred))
y_trainpred = rf2.predict(X_train)
print(r2_score(y_train, y_trainpred))<jupyter_output><empty_output><jupyter_text>Model 7: Polynomial regression<jupyter_code>poly = PolynomialFeatures(2)
X_poly = poly.fit_transform(X)

X_poly_train, X_poly_test, y_poly_train, y_poly_test = train_test_split(X_poly, y, test_size=0.2)
plr = LinearRegression()
plr.fit(X_poly_train, y_poly_train)

filename = 'models/plr01.sav'
pickle.dump(plr, open(filename, 'wb'))
plr.score(X_poly_test, y_poly_test)
plr.score(X_poly_train, y_poly_train)<jupyter_output><empty_output><jupyter_text>This doesn't seem to be overfitting. Maybe try higher-order polynomial features  Model 8: Polynomial regression, third-order polynomials<jupyter_code>poly3 = PolynomialFeatures(3)
X_poly3 = poly.fit_transform(X)

X_poly3_train, X_poly3_test, y_poly3_train, y_poly3_test = train_test_split(X_poly3, y, test_size=0.2)
plr2 = LinearRegression()
plr2.fit(X_poly_train, y_poly_train)

filename = 'models/plr02.sav'
pickle.dump(plr2, open(filename, 'wb'))
plr.score(X_poly3_test, y_poly3_test)
plr.score(X_poly3_train, y_poly3_train)<jupyter_output><empty_output><jupyter_text>Still not overfitting, but isn't doing any better. Try a different approach.  Model 9: Random forest grid search with cross-validation: All features<jupyter_code>df = pd.read_csv('data/model_df_weather.csv', index_col=0)
cols = ['carrier_speed_rank', 'flight_num_speed_rank', 'month_rank',
       'dep_hour_rank', 'arr_hour_rank', 'precip_cat', 'snow_cat',
        'windgust_cat', 'cloud_cat', 'fl_month', 'airline_delay', 
        'haul_length', 'dep_timeday', 'arr_timeday', 'busy_origin', 'busy_dest']
X = df[cols].to_numpy()
y = df['arr_delay'].to_numpy()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
grid = {'n_estimators': [10, 50],
       'max_depth': [None, 5, 20],
#        'min_samples_leaf': [1, 10, 100, 1000]
       }

gs = GridSearchCV(estimator=RandomForestRegressor(), param_grid=grid)
gs.fit(X_train, y_train)
print(gs.best_params_)
print(gs.best_score_)
model = gs.best_estimator_
filename = 'models/rf3.sav'
pickle.dump(model, open(filename, 'wb'))
filename = 'models/gs1.sav'
pickle.dump(gs, open(filename, 'wb'))<jupyter_output><empty_output><jupyter_text>Model 10: Logistic Regression - Baseline<jupyter_code>from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1317016317016317, 'fraction_numerical': 0.023776223776223775, 'mean_word_length': 7.513888888888889, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '68206', 'license': 'MIT', 'n_tokens_mistral': 1764, 'n_tokens_neox': 1678, 'n_words': 289, 'path': 'Oliver_workspace/MODELS.ipynb', 'repo_name': 'robynmundle/predicting_flight_delays', 'size': '4293'}