<jupyter_start><jupyter_text>Training<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import GridSearchCV

import sys
sys.path.append('..')

import src.utils as utils
%load_ext autoreload
%autoreload 2
FILE_PATH = '../data/processed/eo_football_processed.csv'
football = pd.read_csv(FILE_PATH)
football.head()
football['Date'] = pd.to_datetime(football['Date'])
football.head()
# grab six weeks of every season
data = utils.rows_extractor(football, shift=8)
data.to_csv('data.csv', index=False)
data.head()
label_encoder = LabelEncoder()
data['FTR'] = label_encoder.fit_transform(data.FTR)
X = data.iloc[:,5:]
y = data['FTR']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42)
model = RandomForestClassifier(n_estimators=300, max_depth=32, min_samples_split=100)
model.fit(X_train, y_train)
model.score(X_train, y_train)
model.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Optimization<jupyter_code>import optuna
def objective(trial):    
    
    n_estimators = trial.suggest_int('n_estimators', 100, 1000)
    max_depth = int(trial.suggest_float('max_depth', 1, 32, log=True))
    min_samples_split = int(trial.suggest_float('min_samples_split', 200, 1000, log=True))

    clf = RandomForestClassifier(    
        n_estimators=n_estimators, max_depth=max_depth, min_samples_split=min_samples_split)
    
    return cross_val_score(
        clf, X_train, y_train, n_jobs=-1, cv=10).mean()  # Train and evaluate the model.

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100)

trial = study.best_trial

print('Accuracy: {}'.format(trial.value))
print("Best hyperparameters: {}".format(trial.params))
model = RandomForestClassifier(n_estimators=647, max_depth=8, min_samples_split=203)
model.fit(X_train, y_train)
model.score(X_train, y_train)
model.score(X_test, y_test)
data_imp = [[col, imp] for col, imp in zip(X_train.columns, model.feature_importances_)]
feature_importance = pd.DataFrame(data, columns=['Column', 'Feature Importance'])
feature_importance.sort_values(by='Feature Importance', inplace=True, ascending=False)
feature_importance
fig, ax = plt.subplots(1,1, figsize=(12, 8))
plt.barh(feature_importance['Column'], feature_importance['Feature Importance'])
plt.show()
X = X[['HELORT', 'AELORT', 'HGD', 'AAVGST', 'HAVGGS']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42)
model = RandomForestClassifier(n_estimators=647, max_depth=8, min_samples_split=203)
model.fit(X_train, y_train)
model.score(X_train, y_train)
model.score(X_test, y_test)
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100)

trial = study.best_trial

print('Accuracy: {}'.format(trial.value))
print("Best hyperparameters: {}".format(trial.params))<jupyter_output>[32m[I 2022-04-08 15:49:22,385][0m A new study created in memory with name: no-name-0280e9fb-3f83-4f80-b4fc-a594868e53ff[0m
[32m[I 2022-04-08 15:49:44,942][0m Trial 0 finished with value: 0.4588709677419354 and parameters: {'n_estimators': 438, 'max_depth': 4.448034138881384, 'min_samples_split': 940.5536621548246}. Best is trial 0 with value: 0.4588709677419354.[0m
[32m[I 2022-04-08 15:49:45,772][0m Trial 1 finished with value: 0.4588709677419354 and parameters: {'n_estimators': 100, 'max_depth': 9.062827755715574, 'min_samples_split': 827.8761787354051}. Best is trial 0 with value: 0.4588709677419354.[0m
[32m[I 2022-04-08 15:49:49,094][0m Trial 2 finished with value: 0.5395161290322581 and parameters: {'n_estimators': 389, 'max_depth': 4.0032786630663955, 'min_samples_split': 568.736441323454}. Best is trial 2 with value: 0.5395161290322581.[0m
[32m[I 2022-04-08 15:49:51,692][0m Trial 3 finished with value: 0.5370967741935483 and parameters: {'n_estimators': 314, 'max_d[...]
{'fraction_non_alphanumeric': 0.1336081474296799, 'fraction_numerical': 0.10184287099903007, 'mean_word_length': 7.233532934131737, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '587777', 'license': 'MIT', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1541, 'n_words': 306, 'path': 'football-predictions/notebooks/s.o-training.ipynb', 'repo_name': 'samie-hash/data-science-repo', 'size': '4124'}