<jupyter_start><jupyter_text>Modelling<jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.pipeline import Pipeline

from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix

import matplotlib.pyplot as plt
def npv_score(y_test, y_pred):
    true_negatives = np.unique(np.array(list(zip(y_test.values, y_pred))).sum(axis=1), return_counts=True)[1][0]
    predicted_negatives = np.unique(y_pred, return_counts=True)[1][0]
    npv = true_negatives / predicted_negatives
    return npv
df = pd.read_csv("data/games_with_features.csv", index_col="id")
df.head()<jupyter_output><empty_output><jupyter_text>Ultra Baseline<jupyter_code># predict winner only using home_team_avg_score when playing at home
# vs visitor_team_avg_score when playing as visitor
home_should_win = df[df["home_team_avg_score"].gt(df["visitor_team_avg_score"])]
home_should_win["winner"].value_counts(normalize=True)
s2018 = df[df["season"].eq(2018)]
# predict winner only using home_team_avg_score when playing at home
# vs visitor_team_avg_score when playing as visitor
home_should_win = s2018[s2018["home_team_avg_score"].gt(s2018["visitor_team_avg_score"])]
home_should_win["winner"].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>> Home team wins 66% of the time when their avg score is higher   > there is a flaw with this baseline in that it uses the average score from all games   > that occured that season and correlates it with the winner of games that occured before   > that average score was known<jupyter_code>df[(df["home_team_avg_score"] - 5).gt(df["visitor_team_avg_score"])]["winner"].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>> same experiment as above but only using teams that are heavy favourites (10 pt average more).  > win percantage seems to increse 1% per 1pt advantage<jupyter_code>s2020 = df[df["season"].eq(2020)]
s2019 = df[df["season"].eq(2019)]
s2018 = df[df["season"].eq(2018)]
s2017 = df[df["season"].eq(2017)]
s2016 = df[df["season"].eq(2016)]
s2015 = df[df["season"].eq(2015)]<jupyter_output><empty_output><jupyter_text>Baseline<jupyter_code>lr = LogisticRegression(max_iter=1000)
nb = GaussianNB()
knn = KNeighborsClassifier()
rf = RandomForestClassifier()
xgb = XGBClassifier(eval_metric="logloss", use_label_encoder=False)

models = [lr, nb, knn, rf, xgb]
def train_model(model, train_data, test_data):
    X_train = train_data.drop("winner", axis=1)
    y_train = train_data["winner"]
    X_test = test_data.drop("winner", axis=1)
    y_test = test_data["winner"]                     

    model = model
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)

    print(accuracy_score(y_test, y_pred))
    #print(cross_val_score(model, feats, target).mean())
train_data = df[df["season"].isin([2017])][["home_team_avg_score", "visitor_team_avg_score", "winner"]]
test_data = df[df["season"].isin([2018])][["home_team_avg_score", "visitor_team_avg_score", "winner"]]
for model in models:
    train_model(model, train_data, test_data)<jupyter_output>0.6399694889397407
0.6414950419527079
0.5713196033562167
0.5530129672006102
0.5408085430968727<jupyter_text>using score diff feat<jupyter_code>features = ["winner", "home_team_avg_score", "visitor_team_avg_score", "home_avg_score_diff", "visitor_avg_score_diff"]
train_data = df[df["season"].isin(range(2015,2018))][features]
test_data = df[df["season"].isin([2018])][features]
for model in models:
    train_model(model, train_data, test_data)<jupyter_output>0.6361556064073226
0.6376811594202898
0.566742944317315
0.6064073226544623
0.5881006864988558<jupyter_text>> xgboost and random forest predict around 80% correctly when using data from the same season.  > unfortunately when trying to predict results of future seasons the result is horrible.  The Big problem with this so far is that averages for the season are used to predict outcomes of games that contributed to those averages  2020 stats with test info leaking into train set<jupyter_code>stats = pd.read_csv("data/stats_feats2020.csv", index_col="game_id")
X = stats.drop(["winner", "game_date", "season"], axis=1)
y = stats["winner"]

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75)

for model in models:
    model = model
    model.fit(X_train, y_train)
    y_train_pred = model.predict(X_train)
    y_pred = model.predict(X_test)
    
    print(model)
    print(accuracy_score(y_train, y_train_pred))
    print(accuracy_score(y_test, y_pred))
    print(cross_val_score(model, X_test, y_test).mean())<jupyter_output>LogisticRegression(max_iter=1000)
0.8137603795966786
0.8156028368794326
0.8152882205513784
GaussianNB()
0.7663107947805456
0.7624113475177305
0.7197994987468672
KNeighborsClassifier()
0.8054567022538552
0.7127659574468085
0.6771303258145364
RandomForestClassifier()
1.0
0.74822695035461
0.7197994987468672
XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, eval_metric='logloss',
              gamma=0, gpu_id=-1, importance_type='gain',
              interaction_constraints='', learning_rate=0.300000012,
              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,
              monotone_constraints='()', n_estimators=100, n_jobs=8,
              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,
              scale_pos_weight=1, subsample=1, tree_method='exact',
              use_label_encoder=False, validate_parameters=1, verbosity=None)
1.0
0.7624113475177305
0.7199874686716792<jupyter_text>> this seems to good to be true     Using stats from 2015 - 2020<jupyter_code>lr = LogisticRegression(max_iter=1000)
nb = GaussianNB()
knn = KNeighborsClassifier()
rf = RandomForestClassifier(n_estimators=100)
xgb = XGBClassifier(n_estimators=100, eval_metric="logloss", use_label_encoder=False)

models = [lr, nb, knn, rf, xgb]
stats = pd.read_csv("data/stats_feats.csv", index_col="game_id")
X = stats.drop(["winner", "game_date", "season"], axis=1)
y = stats["winner"]

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75)

for model in models:
    model = model
    model.fit(X_train, y_train)
    y_train_pred = model.predict(X_train)
    y_pred = model.predict(X_test)
    
    print(model)
    print(accuracy_score(y_train, y_train_pred))
    print(accuracy_score(y_test, y_pred))
    print(cross_val_score(model, X_test, y_test).mean())<jupyter_output>/Users/zacharyargentin/opt/anaconda3/envs/bootcamp/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(<jupyter_text>> The test data is no longer leaking into the training data. But the model still overfits<jupyter_code>train = stats[stats["season"].isin([2015,2016,2017,2018,2019])]
test = stats[stats["season"].eq(2020)]
X_train = train.drop(["winner", "game_date", "season"], axis=1)
y_train = train["winner"]
X_test = test.drop(["winner", "game_date", "season"], axis=1)
y_test = test["winner"]
for model in models:
    model = model
    model.fit(X_train, y_train)
    y_train_pred = model.predict(X_train)
    y_pred = model.predict(X_test)
    
    print(model)
    print(accuracy_score(y_train, y_train_pred))
    print(accuracy_score(y_test, y_pred))
    print(cross_val_score(model, X_test, y_test).mean())<jupyter_output>LogisticRegression(max_iter=1000)
0.6329042029534268
0.570446735395189<jupyter_text>Using all stats<jupyter_code>lr = LogisticRegression(max_iter=5000)
nb = GaussianNB()
knn = KNeighborsClassifier()
rf = RandomForestClassifier(n_estimators=100)
xgb = XGBClassifier(n_estimators=100, eval_metric="logloss", use_label_encoder=False)

models = [lr, nb, knn, rf, xgb]
stats = pd.read_csv("data/stats_feats.csv", index_col="game.id")
X = stats.drop(["winner", "game.date", "game.season"], axis=1)
y = stats["winner"]

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75)

for model in models:
    model = model
    model.fit(X_train, y_train)
    y_train_pred = model.predict(X_train)
    y_pred = model.predict(X_test)
    
    print(model)
    print(accuracy_score(y_train, y_train_pred))
    print(accuracy_score(y_test, y_pred))
    print(cross_val_score(model, X_test, y_test).mean())<jupyter_output>LogisticRegression(max_iter=5000)
0.6431574583999526
0.6485478284039435
0.6481918436881812
GaussianNB()
0.6046367027891277
0.6144417799094058
0.617460761469345
KNeighborsClassifier()
0.738052940131462
0.6019184652278178
0.5929479536625917
RandomForestClassifier()
1.0
0.6354027888800071
0.6329151125254772
XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, eval_metric='logloss',
              gamma=0, gpu_id=-1, importance_type='gain',
              interaction_constraints='', learning_rate=0.300000012,
              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,
              monotone_constraints='()', n_estimators=100, n_jobs=8,
              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,
              scale_pos_weight=1, subsample=1, tree_method='exact',
              use_label_encoder=False, validate_parameters=1, verbosity=None)
0.8328003789897555
0.632116528999023
0.62145890557[...]<jupyter_text>Using stats from 2020 season<jupyter_code>train = stats[stats["season"].lt(2020)]
test = stats[stats["season"].eq(2020)]
X_train = train.drop(["winner", "game_date", "season"], axis=1)
y_train = train["winner"]
X_test = test.drop(["winner", "game_date", "season"], axis=1)
y_test = test["winner"]
# using only diff
X_train = X_train.loc[:,"diff_away_ast":]
X_test = X_test.loc[:,"diff_away_ast":]
for model in models:
    model = model
    model.fit(X_train, y_train)
    y_train_pred = model.predict(X_train)
    y_pred = model.predict(X_test)
    
    print(model)
    print(accuracy_score(y_train, y_train_pred))
    print(accuracy_score(y_test, y_pred))
    print(precision_score(y_test, y_pred))
    print(recall_score(y_test, y_pred))
    print(confusion_matrix(y_test, y_pred))<jupyter_output>LogisticRegression(max_iter=5000)
0.6185157137447936
0.5747422680412371
0.5902688860435339
0.7248427672955975
[[208 320]
 [175 461]]
GaussianNB()
0.5808405906853464
0.5567010309278351
0.5742574257425742
0.7295597484276729
[[184 344]
 [172 464]]
KNeighborsClassifier()
0.7288905717531238
0.5042955326460481
0.5439642324888226
0.5738993710691824
[[222 306]
 [271 365]]
RandomForestClassifier()
1.0
0.538659793814433
0.558858501783591
0.7389937106918238
[[157 371]
 [166 470]]
XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
              colsample_bynode=1, colsample_bytree=1, eval_metric='logloss',
              gamma=0, gpu_id=-1, importance_type='gain',
              interaction_constraints='', learning_rate=0.300000012,
              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,
              monotone_constraints='()', n_estimators=100, n_jobs=8,
              num_parallel_tree=1, random_state=0, reg_alpha=0, reg_lambda=1,
              scale_pos_weig[...]<jupyter_text>> More data slightly improved random forest and XGboost algos.   > Some models overfit so possibly hyper parameter tuning would help?   > But I think we need to be more selective of features / engineer better featrures  best_model<jupyter_code>lr.fit(X_train, y_train)
y_train_pred = lr.predict(X_train)
y_pred = lr.predict(X_test)

print(accuracy_score(y_train, y_train_pred))
print(accuracy_score(y_test, y_pred))
print(precision_score(y_test, y_pred))
print(npv_score(y_test, y_pred))
print(confusion_matrix(y_test, y_pred))
plt.figure()
plot_confusion_matrix(lr, X_test, y_test)
plt.ylabel("True Outcome")
plt.yticks(ticks=[0,1], labels=["Away Win", "Home Win"])
plt.xticks(ticks=[0,1], labels=["Away Win", "Home Win"])
plt.xlabel("Predicted Outcome")
plt.show()
176/311<jupyter_output><empty_output><jupyter_text>Saving the best model<jupyter_code>import pickle
NBA_results_predictor = lr.fit(X, y)

pickle.dump(NBA_results_predictor, open("model.sav", "wb"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11726309322684687, 'fraction_numerical': 0.09821839963139303, 'mean_word_length': 6.001612903225807, 'pattern_counts': {'":': 2, '<': 46, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '571171', 'license': 'MIT', 'n_tokens_mistral': 5549, 'n_tokens_neox': 4780, 'n_words': 891, 'path': '4_modelling.ipynb', 'repo_name': 'ZacharyArgentin/Predicting_NBA_Results', 'size': '13022'}