<jupyter_start><jupyter_text>Create the columns 'year', 'month', 'day' separating the 'fl_date' column and after that conert it back to a datetime format<jupyter_code>df[["year", "month" ,"day"]] = df["fl_date"].str.split("-", expand = True)
df1 = df.pop('year')
df2 = df.pop('month')
df3 = df.pop('day')
df.insert(1, "year", df1)
df.insert(2, "month", df2)
df.insert(3, "day", df3)
df['date'] = pd.to_datetime(df[['year','month', 'day']])
df.drop(['fl_date'], axis=1)
import datetime

#_________________________________________________________
# Function that convert the 'HHMM' string to datetime.time "1940 -> 19:40:00"
def time_format(chain):
    if pd.isnull(chain):
        return np.nan
    else:
        if chain == 2400: chain = 0
        chain = "{0:04d}".format(int(chain))
        time = datetime.time(int(chain[0:2]), int(chain[2:4]))
        return time
#_____________________________________________________________________
# Function that combines a date and time to produce a datetime.datetime
def combine_date_time(x):
    if pd.isnull(x[0]) or pd.isnull(x[1]):
        return np.nan
    else:
        return datetime.datetime.combine(x[0],x[1])
#_______________________________________________________________________________
# Function that combine two columns of the dataframe to create a datetime format "1940 -> 2018-01-01 19:40:00"
def create_flight_time(df, col):    
    list1 = []
    for index, cols in df[['date', col]].iterrows():    
        if pd.isnull(cols[1]):
            list1.append(np.nan)
        elif float(cols[1]) == 2400:
            cols[0] += datetime.timedelta(days=1)
            cols[1] = datetime.time(0,0)
            list1.append(combine_date_time(cols))
        else:
            cols[1] = format_heure(cols[1])
            list1.append(combine_date_time(cols))
    return pd.Series(list1)
# df['crs_dep_time'] = create_flight_time(df, 'crs_dep_time')
df['crs_dep_time'] = df['crs_dep_time'].apply(time_format)
df['dep_time'] = df['dep_time'].apply(time_format)
df['crs_arr_time'] = df['crs_arr_time'].apply(time_format)
df['arr_time'] = df['arr_time'].apply(time_format)
variables_to_remove = ['taxi_out', 'taxi_in', 'wheels_on', 'wheels_off', 'year','month','day','date', 'nas_delay', 'security_delay', 'carrier_delay', 'late_aircraft_delay',
                       'weather_delay', 'diverted', 'cancelled', 'cancellation_code','mkt_carrier_fl_num', 'tail_num', 'air_time', 'mkt_unique_carrier', 'branded_code_share', 
                       'mkt_carrier', 'op_carrier_fl_num', 'dup', 'flights', 'first_dep_time', 'total_add_gtime', 'longest_add_gtime', 'no_name']
df.drop(variables_to_remove, axis = 1, inplace = True)
df = df[['op_unique_carrier', 'origin_city_name', 'origin', 'origin_airport_id', 'dest_city_name', 'dest', 'dest_airport_id', 'crs_dep_time', 'dep_time', 'dep_delay', 'crs_arr_time', 'arr_time', 'arr_delay', 'crs_elapsed_time', 'actual_elapsed_time']]
df
df.dropna(inplace = True)
missing_df = df.isnull().sum(axis=0).reset_index()
missing_df.columns = ['variable', 'missing values']
missing_df['filling factor (%)']=(df.shape[0]-missing_df['missing values'])/df.shape[0]*100
missing_df.sort_values('filling factor (%)').reset_index(drop = True)
#__________________________________________________________________
# function that extract statistical parameters from a grouby objet:
def get_stats(group):
    return {'min': group.min(), 'max': group.max(),
            'count': group.count(), 'mean': group.mean()}
#_______________________________________________________________
# Creation of a dataframe with statitical infos on each airline:
global_stats = df['dep_delay'].groupby(df['op_unique_carrier']).apply(get_stats).unstack()
global_stats = global_stats.sort_values('count')
global_stats
df
from sklearn.preprocessing import StandardScaler
#grouping of flights by departure time and averaging
test2 = df['arr_delay'].groupby(df['dep_time']).apply(get_stats).unstack()
test2.reset_index(inplace=True)

# convert from hour to minutes
fct = lambda x:x.hour*60+x.minute
test2.reset_index(inplace=True)
test2['dep_time_min'] = test2['dep_time'].apply(fct)
test2
from sklearn.preprocessing import PolynomialFeatures

test = test2[['mean', 'dep_time_min']].dropna(how='any', axis = 0)

X = np.array(test['dep_time_min'])
Y = np.array(test['mean'])

X = X.reshape(len(X),1)
Y = Y.reshape(len(Y),1)

poly = PolynomialFeatures(2)
Xt = poly.fit_transform(X)
X_test, X_train, y_test2, y_train = train_test_split(Xt,Y, train_size=0.8, random_state=1234)

model = LinearRegression()
model.fit(X_train, y_train)

y_pred2 = model.predict(X_test)
r2_train = model.score(X_train, y_train)
r2_test = model.score(X_test, y_test2)

print("---Results---")
print(f"degree = {2}")
print(f"Train score = {r2_train}")
print(f"Test score = {r2_test}")
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test2, y_pred2))
print('Mean Squared Error:', metrics.mean_squared_error(y_test2, y_pred2))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test2, y_pred2)))
# scaler = StandardScaler()
# X_scaled = scaler.fit_transform(X)
# y_scaled = scaler.fit_transform(Y.reshape(-1,1))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15745909528392685, 'fraction_numerical': 0.021944177093359, 'mean_word_length': 5.713178294573644, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '107724', 'license': 'MIT', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1785, 'n_words': 350, 'path': '.ipynb_checkpoints/data_train_test_poly-checkpoint.ipynb', 'repo_name': 'ZacharyArgentin/LHL_Midterm_Project_Predicting_Flight_Delays', 'size': '5195'}