<jupyter_start><jupyter_text>MLB's Biggest All-Star Injustices<jupyter_code># Import dependencies
import numpy as np
import pandas as pd
pd.set_option('display.max_columns', 100)
pd.options.mode.chained_assignment = None

from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.preprocessing import StandardScaler

import seaborn as sns
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
# Print accuracy of pandas crosstab
def crosstabAccuracy(ct):
    try:
        acc = (ct[0][0]+ct[1][1]) / (ct[0][0]+ct[1][1]+ct[0][1]+ct[1][0])
    except:
        acc = (ct[0][0]) / (ct[0][0]+ct[1][0])
    return(100*round(acc,3))
# Print classification report with specified threshold
def thresholdReport(continuous_predictions, actual_results, threshold):
    updated_preds = np.array([1 if pred > threshold else 0 for pred in continuous_predictions])
    print(classification_report(y_pred=updated_preds, y_true=actual_results))
    print(pd.crosstab(updated_preds, actual_results))
# Read data
fh = pd.read_csv('.\\data\\firsthalf.csv')

# Change 'position' to dummy variables
position_dummies = pd.get_dummies(fh.position)
fh = fh.drop('position', axis=1)
fh = pd.concat([fh, position_dummies], axis=1)
# Initial df metrics
print(fh.shape)
print(fh.made_asg.value_counts(normalize=True))
print(fh.columns)
# Set features
modelcols = [
    'AVG',
    'Def',
    'HR',
    'K%',
    'SB',
    'WAR',
    'popular',
    'won_WS_PY',
    'lost_WS_PY',
    '1B',
    '2B',
    '3B',
    'C',
    'DH',
    'OF',
    'SS'
]

Y = fh.made_asg
X = fh.loc[:,modelcols]
# Correlation matrix
sns.heatmap(X.corr(), cmap='RdBu_r')
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.20, 
                                                stratify=Y, random_state=1000)<jupyter_output><empty_output><jupyter_text>Logistic Regression (unscaled)<jupyter_code># Train logistic regression model
logR = LogisticRegression(penalty='l1')
logR.fit(Xtrain, Ytrain)
pd.crosstab(logR.predict(Xtrain), Ytrain)
# Test logistic regression model
logR_preds_binary = logR.predict(Xtest)
logR_preds_continuous = logR.predict_proba(Xtest)[:,1]
logR_ct = pd.crosstab(logR_preds_binary, Ytest)
print('Accuracy:',crosstabAccuracy(logR_ct))
print('AUC: {:.1f}'.format(100*roc_auc_score(y_score=logR_preds_continuous, y_true=Ytest)))
logR_ct
# Classification report @ 0.40 threshold
thresholdReport(continuous_predictions=logR_preds_continuous,
                actual_results=Ytest, 
                threshold=0.40)
# Feature coefficients
print(len(X.columns), 'features:')
for num, feature in enumerate(Xtrain.columns):
    print(logR.coef_[0][num], feature)<jupyter_output>16 features:
12.379747894756527 AVG
-0.08465519851976787 Def
0.13280894704336563 HR
-0.08212782124728421 K%
0.03243155022366087 SB
1.1514895480457437 WAR
0.3034187139195672 popular
0.994338258269651 won_WS_PY
1.0377194586969876 lost_WS_PY
-1.1061999746345663 1B
0.0 2B
-0.2507775754295886 3B
1.1517490471387446 C
-1.16989673207255 DH
-0.8091043195673008 OF
0.5604062178846984 SS<jupyter_text>Lasso / Ridge / Elastic Net<jupyter_code># Scale all features for lasso, ridge, EN
scaler = StandardScaler()
Xtrainscaled = pd.DataFrame(scaler.fit_transform(Xtrain))
Xtrainscaled.columns = modelcols
Xtestscaled = pd.DataFrame(scaler.transform(Xtest))
Xtestscaled.columns = modelcols
# Binary columns back to 0-1
binaries = ['popular', 'NYY', 'BOS', 'CHC', 'LAD', 'won_WS_PY',
            'lost_WS_PY', 'played_WS_PY', '1B', '2B', '3B', 'C',
            'DH', 'OF', 'SS']
for col in binaries:
    try:
        Xtrainscaled[col] = Xtrainscaled[col].apply(lambda x: 1 if x>0 else 0)
        Xtestscaled[col] = Xtestscaled[col].apply(lambda x: 1 if x>0 else 0)
    except:
        pass
# Conduct Lasso, Ridge, EN for different levels of alpha (never outperforms logistic)
print('AUCs:\n\n')
for i in np.arange(0.01, 0.50, 0.02):
    alpha = i
    print('Alpha = {:.2f}'.format(alpha))
    
    lasso_model = SGDClassifier(penalty='l1', alpha=alpha, max_iter=100, loss='modified_huber')
    lasso_model.fit(Xtrainscaled, Ytrain)
    ridge_model = SGDClassifier(penalty='l2', alpha=alpha, max_iter=100, loss='modified_huber')
    ridge_model.fit(Xtrainscaled, Ytrain)
    elastic_model = SGDClassifier(penalty='l1', alpha=alpha, l1_ratio=0.50, max_iter=100, loss='modified_huber')
    elastic_model.fit(Xtrainscaled, Ytrain)

    lasso_model_preds = lasso_model.predict_proba(Xtestscaled)[:,1]
    print('Lasso: {:.1f}'.format(100*roc_auc_score(y_score=lasso_model_preds, y_true=Ytest)))

    ridge_model_preds = ridge_model.predict_proba(Xtestscaled)[:,1]
    print('Ridge: {:.1f}'.format(100*roc_auc_score(y_score=ridge_model_preds, y_true=Ytest)))    

    elastic_model_preds = elastic_model.predict_proba(Xtestscaled)[:,1]
    print('Elastic: {:.1f}'.format(100*roc_auc_score(y_score=elastic_model_preds, y_true=Ytest)))
    
    print('------------')<jupyter_output>AUCs:


Alpha = 0.01
Lasso: 91.3
Ridge: 92.2
Elastic: 91.6
------------
Alpha = 0.03
Lasso: 90.4
Ridge: 92.0
Elastic: 90.6
------------
Alpha = 0.05
Lasso: 90.2
Ridge: 91.9
Elastic: 90.2
------------
Alpha = 0.07
Lasso: 90.3
Ridge: 91.9
Elastic: 90.1
------------
Alpha = 0.09
Lasso: 90.1
Ridge: 91.8
Elastic: 90.3
------------
Alpha = 0.11
Lasso: 90.1
Ridge: 91.7
Elastic: 90.1
------------
Alpha = 0.13
Lasso: 90.1
Ridge: 91.6
Elastic: 90.1
------------
Alpha = 0.15
Lasso: 90.1
Ridge: 91.5
Elastic: 90.1
------------
Alpha = 0.17
Lasso: 90.1
Ridge: 91.5
Elastic: 90.1
------------
Alpha = 0.19
Lasso: 90.0
Ridge: 91.4
Elastic: 89.2
------------
Alpha = 0.21
Lasso: 89.9
Ridge: 91.4
Elastic: 90.0
------------
Alpha = 0.23
Lasso: 89.9
Ridge: 91.3
Elastic: 89.9
------------
Alpha = 0.25
Lasso: 89.1
Ridge: 91.3
Elastic: 88.7
------------
Alpha = 0.27
Lasso: 89.0
Ridge: 91.3
Elastic: 89.7
------------
Alpha = 0.29
Lasso: 89.5
Ridge: 91.2
Elastic: 89.5
------------
Alpha = 0.31
Lasso: 88.7
Ridge: [...]<jupyter_text>Random Forest<jupyter_code># Grid search for random forest
params = {
    'max_depth':[5,6,7,8],
    'max_features':[3,5,10,None],
    'min_samples_leaf':[1,3,7,11],
    'n_estimators':[301]   
}

rf_for_gs = RandomForestClassifier()
grid_search_rf = GridSearchCV(estimator=rf_for_gs, param_grid=params, cv=7, n_jobs=4)
grid_search_rf.fit(Xtrain, Ytrain)
# Best random forest parameters
grid_search_rf.best_params_
# Train model
rf = RandomForestClassifier(max_depth=8, max_features=5, min_samples_leaf=3, 
                            n_estimators=1001, oob_score=True)
rf.fit(Xtrain, Ytrain)
# Training results
pd.crosstab(rf.predict(Xtrain), Ytrain)
# Test results (does not outperform logistic)
rf_probs_binary = rf.predict(Xtest)
rf_probs_continuous = rf.predict_proba(Xtest)[:,1]
ct_rf = pd.crosstab(rf_probs_binary, Ytest)
print('Accuracy: {:.1f}'.format(crosstabAccuracy(ct_rf)))
print('AUC: {:.1f}'.format(100*roc_auc_score(y_score=rf_probs_continuous, y_true=Ytest)))
ct_rf<jupyter_output>Accuracy: 87.8
AUC: 91.5<jupyter_text>Full model (Logistic Regression)<jupyter_code># Train logistic regression on full data set
logR_full = LogisticRegression(penalty='l1').fit(X,Y)
full_preds_lr = pd.Series(logR_full.predict_proba(X)[:,1])
fh_preds = pd.concat([fh, full_preds_lr], axis=1).rename(columns={0:'pred_lr'})
# Feature coefficients
for num, feature in enumerate(X.columns):
    print(round(logR_full.coef_[0][num],2), feature)
# Reorder columns
cols = fh_preds.columns.tolist()
cols.insert(1, cols.pop(cols.index('year')))   # move "year"
cols.insert(2, cols.pop(cols.index('made_asg')))   # move "made_asg"
cols.insert(3, cols.pop(cols.index('started_asg')))   # move "started_asg"
cols = cols[-1:] + cols[:-1]
fh_preds = fh_preds[cols]
# Should have made ASG, but didn't
fh_preds[fh_preds['made_asg']==0].sort_values('pred_lr', ascending=False).head(5)
# Made ASG, but shouldn't have
fh_preds[fh_preds['made_asg']==1].sort_values('pred_lr', ascending=True).head(5)
fh_preds.sort_values('pred_lr', ascending=False).tail(5)<jupyter_output><empty_output><jupyter_text>Deploy model on 2018 first-half data<jupyter_code># Import 2018 data
fh2018_full = pd.read_csv('.\\data\\firsthalf2018.csv')

# Change 'position' to dummy variables
position_dummies2 = pd.get_dummies(fh2018_full.position)
fh2018_full = fh2018_full.drop(['position', 'Unnamed: 0'], axis=1)
fh2018_full = pd.concat([fh2018_full, position_dummies2], axis=1)
# Deploy logistic regression model on 2018 data
fh2018 = fh2018_full.loc[:,modelcols]
fh2018_full['prob_lr'] = pd.Series(logR_full.predict_proba(fh2018)[:,1])
# Lowest 2018 ASG probabilities
fh2018_full.loc[:,['player', 'prob_lr', 'AVG', 'OBP', 'SLG', 'HR', 'WAR']].sort_values('prob_lr', ascending=True).head(5)
# Highest 2018 ASG probabilities
fh2018_full.loc[:,['player', 'prob_lr', 'AVG', 'OBP', 'SLG', 'HR', 'WAR']].sort_values('prob_lr', ascending=False).head(5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16195401034217186, 'fraction_numerical': 0.07547584992848498, 'mean_word_length': 4.691922354414527, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '11682', 'license': 'MIT', 'n_tokens_mistral': 4036, 'n_tokens_neox': 3538, 'n_words': 667, 'path': 'python/made_ASG.ipynb', 'repo_name': 'saisenberg/mlb-asg', 'size': '9089'}