<jupyter_start><jupyter_text>**Split train and test data for cross-validation**<jupyter_code>X_train, X_test, y_train, y_test = splitByRaceDate(race_rounds_dates[18])
num_cols = len(X_train.columns) -1
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numeric_features_df = X_train.select_dtypes(include=numerics)
numeric_features = list(numeric_features_df.columns)<jupyter_output><empty_output><jupyter_text>**Make costume scoring function**<jupyter_code>def mseScorer(y_true, y_pred):
    y_rank = pd.DataFrame(y_pred).rank(method='first')
    return mean_squared_error(y_true=y_true, y_pred=y_rank)
    
mse_scorer = make_scorer(mseScorer, greater_is_better=False)
def noRetirementsScorer(y_true, y_pred):
    y_true = y_true.reset_index()
    y_rank = pd.DataFrame(y_pred).rank(method='first')
    y_concat = pd.concat([y_true, y_rank], axis=1)
    #print(y_concat.head(20))
    y_concat = y_concat[y_concat['finished'] == True]
    y_final = y_concat.iloc[:,-2:]
    y_final['rank_pred'] = y_final[y_final.columns[1]].rank(method='first')
    return mean_squared_error(y_true=y_final['positionOrder'], y_pred=y_final['rank_pred'])

no_retirements_scorer = make_scorer(noRetirementsScorer, greater_is_better=False)
def gridstartScorer(y_true, y_pred):
    y_concat = pd.concat([y_pred, y_true], axis=1, ignore_index=True)
    y_concat = y_concat.rename(columns={0:'y_pred',1:'y_true'})
    y_concat = y_concat.reset_index(level='finished')
    y_concat = y_concat[y_concat.finished == True]
    y_concat['pred_rank'] = y_concat['y_pred'].rank(method='first')
    return mean_squared_error(y_true=y_concat['y_true'],y_pred=y_concat['pred_rank'])<jupyter_output><empty_output><jupyter_text>**Make timeseries cross-validation split, where each split consists of 20 drivers = 1 race**<jupyter_code>tscv = TimeSeriesSplit(n_splits=55, test_size=20)
def calculateGridstartScore(tscv):
    mse_scores = []
    for _, test_index in tscv.split(X_train):
        X_test_cv = X_train.iloc[test_index]
        y_test_cv = y_train.iloc[test_index]
        score = gridstartScorer(y_test_cv, X_test_cv['gridStart'])
        mse_scores.append(score)

    return (statistics.mean(mse_scores), mse_scores)<jupyter_output><empty_output><jupyter_text>**Calculate the score for the baseline model over the cross-validation races**<jupyter_code>mean_mse_scores, mse_scores = calculateGridstartScore(tscv)
print('Gridstart baseline score: ', mean_mse_scores)
def gridSearchCV_results(X_train, y_train, cv, scorer, pipeline, param_grid, output):
    model = GridSearchCV(
        estimator = pipeline,
        param_grid = param_grid,
        cv=cv,
        n_jobs=-1,
        scoring=scorer,
        verbose=1
    )
    model.fit(X_train,y_train)
    if output:
        print('Best parameters:\n', model.best_params_)
        print('Best mean accuracy score in cross-validation:\n', abs(model.best_score_))
    return model.best_estimator_, model.cv_results_<jupyter_output><empty_output><jupyter_text>**Ridge regression**<jupyter_code>trans = ColumnTransformer(
    [('one_hot', OneHotEncoder(drop=None, handle_unknown='ignore'), ['circuitName']),
    ('k_best',  SelectKBest(f_regression), numeric_features)]
    )
num_cols = len(X_train.columns) -1
ridge_pipeline = Pipeline([
    ('col_trans', trans),
    ('model', Ridge(random_state=42))
])
ridge_param_grid = {
    'col_trans__k_best__k': range(1,num_cols,2),
    'model__alpha':[0.001, 0.005, 0.01, 0.02, 0.03, 0.05, 0.1, 0.15, 0.2, 0.3, 0.5, 1],
}
#pipeline.get_params()
ridgeRegression_model, ridge_cv_results = gridSearchCV_results(X_train, y_train, tscv, no_retirements_scorer, ridge_pipeline, ridge_param_grid, output=True)
ridge_cv_results_df = pd.DataFrame(ridge_cv_results)
best_score_cv = ridge_cv_results_df.loc[ridge_cv_results_df['rank_test_score'] == 1]

best_score_cv = best_score_cv.head(1)

best_score_cv_array = best_score_cv.iloc[:,7:-3].values
# ridge_cv_results_df.loc[ridge_cv_results_df['rank_test_score'] == 1]
number_k_best_features = int(best_score_cv['param_col_trans__k_best__k'])
select = SelectKBest(f_regression, k = number_k_best_features)
z = select.fit_transform(X_train[numeric_features], y_train)

filter = select.get_support()
features = np.array(X_train[numeric_features].columns)
 
print("All features:")
print(features)
 
print("Selected best " + str(number_k_best_features) + ":")
print(features[filter])
x = np.absolute(best_score_cv_array)
y = list(x.flatten())<jupyter_output><empty_output><jupyter_text>**Get statistics for validation set for best model and baseline**<jupyter_code>print('Model cv mean: ', round(np.mean(y),2))
print('Model cv std: ', round(np.std(y),2))

print('Baseline cv mean: ', round(np.mean(mse_scores),2))
print('Baseline cv std: ', round(np.std(mse_scores),2))<jupyter_output>Model cv mean:  11.27
Model cv std:  7.98
Baseline cv mean:  12.42
Baseline cv std:  9.01<jupyter_text>**Plot model and baseline error across races in the validation set**<jupyter_code>fig, ax = plt.subplots()

fig.set_size_inches(10, 8)

ax.plot(y, label='Model prediction')
ax.plot(mse_scores, label='Qualifying prediction')

ax.hlines(np.mean(y), xmin = 0, xmax= 55, colors='tab:blue', linestyles='dashed', label='Model average')
ax.hlines(np.mean(mse_scores), xmin = 0, xmax= 55, colors='tab:orange', linestyles='dashed', label='Baseline average')

plt.xlabel('Fold number', fontsize=20)
plt.ylabel('Error value', fontsize=20)

plt.legend(loc="upper right", prop={'size': 15})


plt.savefig("../reports/figures/cv_results" + ".eps")<jupyter_output>The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.<jupyter_text>Random Forest regressor<jupyter_code># random_forest_pipeline = Pipeline([
#     ('one_hot', one_hot_encoder),
#     ('scaler', StandardScaler()),
#     ('model', RandomForestRegressor(random_state=42))
# ])

# random_forest_param_grid = {
#     'scaler':['passthrough', StandardScaler(with_mean=False)],
#     'model__max_depth':[5,8,10,None],
#     'model__min_samples_split':[2,3,4],
#     'model__min_samples_leaf': [2,3,4],
#     'model__n_estimators':[200],
#     'model__max_features':['auto'],
# },

# random_forest_model = gridSearchCV_results(X_train, y_train, tscv, no_retirements_scorer, random_forest_pipeline, random_forest_param_grid, output=True)<jupyter_output><empty_output><jupyter_text>**Test on 5 last races of 2021 season**<jupyter_code>def createResultsTable(X_test, y_pred, y_test, round):
    X_test = X_test.reset_index()
    # X_test = X_test.reset_index(level=['finished'])
    #X_test = X_test.reset_index(drop=True)

    df_results = pd.concat([X_test[['driverRef','finished','gridStart']], pd.DataFrame(y_pred, columns=['y_pred'])], axis=1)

    df_results['rank_pred'] = df_results['y_pred'].rank(method='first').astype(int)
    df_results = df_results.drop('y_pred', axis=1)

    y_test = y_test.reset_index(level=['driverRef'])

    final_result = pd.merge(df_results, y_test[['driverRef','positionOrder']], on=['driverRef'])
    final_result = final_result.rename(columns={'positionOrder':'rank_true'})
    #final_result['y_pred'] = final_result['y_pred'].round(2)

    final_result = final_result.sort_values(by=['gridStart'])
    final_result['round'] = round
    return final_result
df_every_round_results = pd.DataFrame()
test_results = []

model = ridgeRegression_model

for round, date in race_rounds_dates.items():
    X_train, X_test, y_train, y_test = splitByRaceDate(date)
    model.fit(X_train, y_train)
    
    X_test_current_round = X_test.loc[X_test['round'] == round]
    y_pred_current_round = model.predict(X_test_current_round)
    y_test_current_round = y_test.loc[y_test['round'] == round]

    df_results = createResultsTable(X_test_current_round, y_pred_current_round, y_test_current_round, round)
    df_every_round_results = df_every_round_results.append(df_results)

    model_mse = noRetirementsScorer(y_true=y_test_current_round.iloc[:,:-1], y_pred=y_pred_current_round)
    grid_start_mse = noRetirementsScorer(y_true=y_test_current_round.iloc[:,:-1], y_pred=X_test_current_round['gridStart'].to_numpy())
    #model_mse =  noRetirementsScorer(y_true=df_results['rank_true'], y_pred=df_results['rank_pred'])
    #grid_start_mse = noRetirementsScorer(y_true=df_results['rank_true'], y_pred=df_results['gridStart'])

    test_results.append([round, model_mse, grid_start_mse])

test_results = pd.DataFrame(test_results, columns=['round', 'model_error', 'baseline_error'])
test_results
df_every_round_results.reset_index(drop=True)
df_every_round_results.to_csv('../data/processed/0.4-test_results.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14086776383933708, 'fraction_numerical': 0.015421797675221545, 'mean_word_length': 6.758928571428571, 'pattern_counts': {'":': 1, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '155526', 'license': 'RSA-MD', 'n_tokens_mistral': 3312, 'n_tokens_neox': 3181, 'n_words': 470, 'path': 'notebooks/04-models.ipynb', 'repo_name': 'mariuscaptari/machine-learning-coursework', 'size': '8689'}