<jupyter_start><jupyter_text>Setup<jupyter_code>from collections import Counter
import copy
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from skopt import BayesSearchCV
from skopt.space import Real, Categorical, Integer
import statsmodels.formula.api as smf
import tensorflow
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, LeakyReLU
def analyze_accuracy(y_test, y_pred_prob, y_pred_class):
    naive_pred = [1 for i in range(len(y_pred_class))]
    
    ns_auc = roc_auc_score(y_test, naive_pred)
    md_auc = roc_auc_score(y_test, y_pred_prob)
    print('No Skill: ROC AUC=%.3f' % (ns_auc))
    print('Model   : ROC AUC=%.3f' % (md_auc))
    
    cm = confusion_matrix(y_test, y_pred_class)
    sns.heatmap(cm, annot=True)
    plt.show()
    
    ns_fpr, ns_tpr, _ = roc_curve(y_test, naive_pred)
    lr_fpr, lr_tpr, _ = roc_curve(y_test, y_pred_prob)
    # plot the roc curve for the model
    plt.plot(ns_fpr, ns_tpr, linestyle='--', label='No Skill')
    plt.plot(lr_fpr, lr_tpr, marker='.', label='Model')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    # show the legend
    plt.legend()
    # show the plot
    plt.show()
def plot_calibration(y_test, y_prob):
    fop, mpv = calibration_curve(y_test,y_prob, n_bins=10, normalize=True)
    # plot perfectly calibrated
    plt.plot([0, 1], [0, 1], linestyle='--')
    # plot model reliability
    plt.plot(mpv, fop, marker='.')
    plt.show()
def setup_chess_data():
    df = pd.read_csv("chess_king_rook_dataset.csv")
    
    df['win'] = df['result'] != 'draw'
    df['win'] = df['win'].astype(int)
    
    for col in df.columns:
        if "rank" in col:
            df[col] = df[col].astype(str)

    dummy_df = copy.deepcopy(df)
    del dummy_df['result']
    
    dummy_df = pd.get_dummies(dummy_df)
    for col in dummy_df.columns:
        dummy_df[col] = dummy_df[col].astype(float)
    dummy_df['result'] = df['result']
    
    col_list = []
    for col in dummy_df.columns:
        if col != 'win' and col != 'result':
            col_list.append(col)

    X = dummy_df.loc[:, col_list]
    y = dummy_df['win']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0, stratify=y)
    
    return X_train, X_test, y_train, y_test
X_train, X_test, y_train, y_test = setup_chess_data()<jupyter_output><empty_output><jupyter_text>First NN Model<jupyter_code>model = Sequential()
model.add(Dropout(0.2))
model.add(Dense(12, input_dim=40, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(X_train.values, y_train.values, epochs=100, batch_size=10)
y_pred_prob = model.predict(X_test.values)
y_pred_class = [int(x >= 0.5) for x in y_pred_prob]<jupyter_output><empty_output><jupyter_text>Accuracy and Calibration<jupyter_code>analyze_accuracy(y_test, y_pred_prob, y_pred_class)
plot_calibration(y_test, y_pred_prob)<jupyter_output><empty_output><jupyter_text>Second Model: Leakly Relu<jupyter_code>lrelu = lambda x: tensorflow.keras.activations.relu(x, alpha=0.1)

lr_model = Sequential()
lr_model.add(Dropout(0.2))
lr_model.add(Dense(12, input_dim=40, activation=lrelu))
lr_model.add(Dropout(0.2))
lr_model.add(Dense(8, activation=lrelu))
lr_model.add(Dense(1, activation='sigmoid'))
lr_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
lr_model.fit(X_train.values, y_train.values, epochs=100, batch_size=10)<jupyter_output>Train on 22444 samples
Epoch 1/100
22444/22444 [==============================] - 6s 258us/sample - loss: 0.1798 - accuracy: 0.9225
Epoch 2/100
22444/22444 [==============================] - 6s 265us/sample - loss: 0.1769 - accuracy: 0.9233
Epoch 3/100
22444/22444 [==============================] - 6s 282us/sample - loss: 0.1796 - accuracy: 0.9224
Epoch 4/100
22444/22444 [==============================] - 7s 309us/sample - loss: 0.1798 - accuracy: 0.9228
Epoch 5/100
22444/22444 [==============================] - 6s 266us/sample - loss: 0.1780 - accuracy: 0.9243
Epoch 6/100
22444/22444 [==============================] - 6s 270us/sample - loss: 0.1761 - accuracy: 0.9251
Epoch 7/100
22444/22444 [==============================] - 7s 295us/sample - loss: 0.1797 - accuracy: 0.9239
Epoch 8/100
22444/22444 [==============================] - 6s 273us/sample - loss: 0.1798 - accuracy: 0.9228
Epoch 9/100
22444/22444 [==============================] - 6s 270us/sample - loss: 0.1772 - accuracy: 0.9[...]<jupyter_text>Leaky relu seems to start at a higher level of accuracy, but gains less over time<jupyter_code>y_pred_prob_lr = lr_model.predict(X_test.values)
y_pred_class_lr = [int(x >= 0.5) for x in y_pred_prob_lr]
analyze_accuracy(y_test, y_pred_prob_lr, y_pred_class_lr)
plot_calibration(y_test, y_pred_prob_lr)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17191011235955056, 'fraction_numerical': 0.05655430711610487, 'mean_word_length': 4.980963045912654, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '225510', 'license': 'MIT', 'n_tokens_mistral': 2217, 'n_tokens_neox': 1971, 'n_words': 408, 'path': 'ChessPredictionTF.ipynb', 'repo_name': 'MarshallKrakauer/MovePrediction', 'size': '5340'}