<jupyter_start><jupyter_text>get feature from rpe<jupyter_code>def clean_rpe():
    
    pre = pd.read_csv('./data/rpe.csv',na_values=['None'])
    df = pre
    print('df shape is ', df.shape)
        # traing 
    df.Training = df.Training.replace('Yes', 1)
    df.Training = df.Training.replace('No', 0)
    
    # dummies for SessionType
    dummies= pd.get_dummies(df.SessionType, prefix='SessionType' )
    for col in list((dummies)):
        df.insert(4, col, dummies[col] )
    
    # for numeric cols, replace nan with mean for each player
    numeric_cols = [
        'Duration',
        'RPE',
        'SessionLoad',
        'DailyLoad',
        'AcuteLoad',
        'ChronicLoad',
        'AcuteChronicRatio',
        'ObjectiveRating',
        'FocusRating']
    
    for col in numeric_cols:
        for player in set(df.PlayerID):
            mean_ = (df[df.PlayerID == player][col].mean())
            df.update( df[df.PlayerID == player][col].fillna(mean_)) 
#             df[df.PlayerID == player][col] = df[df.PlayerID == player][col].replace(np.nan, mean_)
    
    
    # dummies for BestOutOfMyself
    dummies= pd.get_dummies(df.BestOutOfMyself, prefix='BestOutOfMyself' )
    for col in list((dummies)):
        df.insert(21, col, dummies[col] )

    df = df.drop(columns = ['SessionType','BestOutOfMyself'])
    df.head()
    return df
rpe = pd.read_csv('./data/rpe.csv',na_values=['None'])
rpe[rpe.DailyLoad.notnull()][rpe.Date == day]
# performance = df.groupby(['PlayerID', 'GameID'])['Performance'].mean()
# performance  = pd.DataFrame(data=performance).reset_index()<jupyter_output><empty_output><jupyter_text>Construct model df<jupyter_code>a = rpe[rpe.DailyLoad.notnull()][['Date','PlayerID']]
from datetime import date
from datetime import datetime,timedelta
a = '2018-07-21'
day ='2018-07-20'
(datetime.strptime(day, "%Y-%m-%d") - timedelta(1)).strftime("%Y-%m-%d")
rpe[rpe.DailyLoad.notnull()][rpe.Date == day_minus]
a['Date'] = pd.to_datetime(a['Date']) - pd.to_timedelta(a['day'], unit='d')
model_df = pd.DataFrame()
for g_id in range(38):
    day = game_date.loc[game_date['GameID']== g_id+1].values[0][1]  # game date

    # get performance metric from gps_df
    # they are from people who actually in the game
    true_label = performance[performance['GameID']==g_id]['catagory']
    
    # merge to get model df
    merged_df = performance[performance['GameID']==g_id].merge(
        wellness[wellness.Date == day].drop(columns = ['Date']),
        left_on=['PlayerID'], right_on=['PlayerID'],suffixes=('_left', '_right'))
    
    day_minus = (datetime.strptime(day, "%Y-%m-%d") - timedelta(1)).strftime("%Y-%m-%d")
    merged_df = merged_df.merge(
        rpe[rpe.DailyLoad.notnull()][rpe.Date == day_minus][['AcuteChronicRatio','PlayerID']],
        left_on=['PlayerID'], right_on=['PlayerID'],suffixes=('_left', '_right'))
    
    
    model_df = model_df.append(merged_df, ignore_index = True, sort=False) # append new ones. 
model_matrix = model_df.values
model_df
model_df['Moni'] = model_df['Fatigue']+model_df['Soreness']+model_df['Desire']+model_df['Irritability']+model_df['SleepQuality']
# model_df = model_df.drop(columns = ['Fatigue','Soreness','Desire','Irritability','SleepQuality'])
# model_df
# model_df.info()<jupyter_output><empty_output><jupyter_text>correlation<jupyter_code>correlation = model_df.corr()
plt.figure(figsize=(10,10))
sns.heatmap(correlation, vmax=1, square=True,annot=True,cmap='cubehelix')<jupyter_output><empty_output><jupyter_text>PCA for visualization<jupyter_code># from sklearn.decomposition import PCA
# pca = PCA(n_components=2)
# principalComponents = pca.fit_transform(model_df.drop(columns = ['catagory']).values)
# principalDf = pd.DataFrame(data = principalComponents
#              , columns = ['principal component 1', 'principal component 2'])

# finalDf = pd.concat([principalDf, model_df[['catagory']]], axis = 1)

# finalDf.update(finalDf['Speed'].astype('category'))
# fig = plt.figure(figsize = (8,8))
# ax = fig.add_subplot(1,1,1) 
# ax.set_xlabel('Principal Component 1', fontsize = 15)
# ax.set_ylabel('Principal Component 2', fontsize = 15)
# ax.set_title('2 component PCA', fontsize = 20)
# targets = finalDf['catagory'].unique().tolist()

# for target in zip(targets):
#     indicesToKeep = finalDf['catagory'] == target
#     ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
#                , finalDf.loc[indicesToKeep, 'principal component 2']
#                , s = 50)
# ax.legend(targets)
# ax.grid()
# pca.explained_variance_ratio_<jupyter_output><empty_output><jupyter_text>PCA for speeding up training   split data<jupyter_code>df = model_df.drop(columns = ['GameID','Performance','Pain','Illness','Menstruation','SleepHours',])
df.head()
from sklearn.model_selection import train_test_split
train_X, test_X, train_y, test_y = train_test_split(
    df.drop(columns = ['catagory','PlayerID']), df.catagory, test_size=0.3, random_state=42)<jupyter_output><empty_output><jupyter_text>standardize<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
# Fit on training set only.
scaler.fit(train_X)
# Apply transform to both the training set and the test set.
train_X = scaler.transform(train_X)
test_X = scaler.transform(test_X)<jupyter_output><empty_output><jupyter_text>PCA to reduce dimension<jupyter_code># from sklearn.decomposition import PCA
# # Make an instance of the Model
# pca = PCA(.95)
# pca.fit(train_X)

# train_X = pca.transform(train_X)
# test_X = pca.transform(test_X)<jupyter_output><empty_output><jupyter_text>train the model: logistic regression CV<jupyter_code># from sklearn.linear_model import LogisticRegressionCV, LogisticRegression

# clf = LogisticRegressionCV(cv = 10,random_state=0,
#                            multi_class='auto')
#                            #solver = 'lbfgs')
# clf.fit(train_X, train_y)
# clf.predict(test_X)

# clf.predict_proba(test_X).shape

# clf.score(test_X, test_y)<jupyter_output><empty_output><jupyter_text>train the model: random forest<jupyter_code>from sklearn.tree import DecisionTreeClassifier

dt_classifier = DecisionTreeClassifier()

dt_classifier.fit(train_X, train_y)

dt_predictions = dt_classifier.predict(test_X)


from sklearn.ensemble import RandomForestClassifier

rf_classifier = RandomForestClassifier(n_estimators=1000)

rf_classifier.fit(train_X, train_y)

rf_predictions = rf_classifier.predict(test_X)

# np.sum(rf_predictions != test_y)/test_y.shape[0] # 38.9% error
print("Error rate is {} ".format(np.sum(rf_predictions != test_y)/test_y.shape[0]))
from sklearn.svm import SVC
clf = SVC(gamma='auto')
clf.fit(train_X, train_y)
clf.score(test_X,test_y)
def tsne(df):

    # df
    # run tsne
    tsne = TSNE(n_components=2, verbose=1, perplexity=30, n_iter=300)
    tsne_results = tsne.fit_transform(df.drop(columns=['catagory','PlayerID']).values)
    # plot 
    df_tsne = df.copy()
    df_tsne['x-tsne'] = tsne_results[:,0]
    df_tsne['y-tsne'] = tsne_results[:,1]
    

    # color by sequences
    chart1 = ggplot( df_tsne, aes(x='x-tsne', y='y-tsne', color='catagory') ) \
            + geom_point(size=75,alpha=0.8) \
            + ggtitle("tSNE dimensions colored by PlayerID")
#     # color by label
#     chart2 = ggplot( df_tsne, aes(x='x-tsne', y='y-tsne', color='PlayerID') ) \
#             + geom_point(size=75,alpha=0.8) \
#             + ggtitle("tSNE dimensions colored by LABEL")
    return chart1
tsne(model_df.drop(columns=['GameID','Performance']))
test_y = test_y.replace('high',1).replace('low',0)
rf_predictions = pd.DataFrame(rf_predictions).replace('high',1).replace('low',0).values
from sklearn import metrics
fpr, tpr, thresholds = metrics.roc_curve(test_y, rf_predictions)

plt.plot(fpr, tpr)
fpr

tpr

# thresholds
from sklearn.metrics import roc_auc_score

dt_auc = roc_auc_score(test_y, dt_classifier.predict_proba(test_X)[:, 1])

dt_auc

rf_auc = roc_auc_score(test_y, rf_classifier.predict_proba(test_X)[:, 1])

rf_auc
# from sklearn.cluster import KMeans

# kmeans = KMeans(n_clusters=3, random_state=0).fit(train_X)
# kmeans.labels_
# labels = kmeans.predict(test_X)

# plt.scatter(test_X[:])

# kmeans.cluster_centers_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15267824608283737, 'fraction_numerical': 0.01226770314587635, 'mean_word_length': 4.737979094076655, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '34263', 'license': 'MIT', 'n_tokens_mistral': 3139, 'n_tokens_neox': 3009, 'n_words': 495, 'path': 'use wellness to predict performance.ipynb', 'repo_name': 'NicolasHu11/MCPY', 'size': '8233'}