<jupyter_start><jupyter_text>Are YOU a Crossfit Games Athlete? This new data will let you know if YOU ARE (a) FIT. https://medium.com/p/e83c04eb783e/edit<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
plt.style.use('fivethirtyeight')
import matplotlib
matplotlib.rcParams['axes.labelsize'] = 14
matplotlib.rcParams['xtick.labelsize'] = 12
matplotlib.rcParams['ytick.labelsize'] = 12
matplotlib.rcParams['text.color'] = 'k'
from pylab import rcParams
rcParams['figure.figsize'] = 25, 10
# configure Pandas to publish table schema and data in a JSON representation
# any dataframe outputted after this will display the Data Explorer
# you can turn this off later if you want to view HTML tables instead
pd.options.display.html.table_schema = True # Data Explorer On!
# the Data Explorer performs visualization in the browser,
# so we must increase the number of rows outputted in order for it to be useful
# be careful! some datasets may be too large (see contraints section)
pd.options.display.max_rows = None
data_path = "/Users/pblsnt/Documents/DataScientist_Nanodegree_Udacity/Data"
crossfit_data = "/crossfit_games_data"
def read_file_csv(filepath, header=0, sep=','):
    return pd.read_csv(filepath_or_buffer=filepath, sep=sep, header=header)
crossfit_open_scores_2019_file = data_path + crossfit_data + "/2019_opens_scores.csv"
crossfit_open_athletes_2019_file = data_path + crossfit_data + "/2019_opens_athletes.csv"
crossfit_games_athletes_2019_file = data_path + crossfit_data + "/2019_games_athletes.csv"
crossfit_open_scores_2020_file = data_path + crossfit_data + "/2020_opens_scores.csv"
crossfit_open_athletes_2020_file = data_path + crossfit_data + "/2020_opens_athletes.csv"<jupyter_output><empty_output><jupyter_text>What does a Crossfit Athlete look like?<jupyter_code>crossfit_open_athletes_2019 = read_file_csv(crossfit_open_athletes_2019_file)
crossfit_open_athletes_2019.head(100)
crossfit_open_athletes_2019['category'] = crossfit_open_athletes_2019['is_scaled']
crossfit_open_athletes_2019.loc[crossfit_open_athletes_2019.is_scaled == 0, 'category'] = 'RX'
crossfit_open_athletes_2019.loc[crossfit_open_athletes_2019.is_scaled == 1, 'category'] = 'Scaled'
crossfit_open_athletes_2019.loc[crossfit_open_athletes_2019.postcompstatus=='accepted', 'category'] = 'Elite'
columns_to_drop = ['competitorname'
                   , 'firstname'
                   , 'lastname'
                   , 'postcompstatus'
                   , 'profilepics3key'
                   , 'countryoforigincode'
                   , 'affiliatename'
                   , 'divisionid'
                   , 'is_scaled'
]
crossfit_open_athletes_2019_filtered = crossfit_open_athletes_2019.drop(columns_to_drop, axis=1)<jupyter_output><empty_output><jupyter_text>Athletes Characteristics & Participation<jupyter_code>top_countries_df = crossfit_open_athletes_2019_filtered.countryoforiginname.value_counts().sort_values(ascending=False).head(10)
crossfit_open_athletes_2019_filtered \
    .loc[~crossfit_open_athletes_2019_filtered.countryoforiginname.isin(top_countries_df.index.tolist()),"countryoforiginname"]= 'Other'
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2,2,figsize=(15,15))
crossfit_open_athletes_2019_filtered.groupby('category').competitorid.count().plot.barh(title='Athlete Level', ax=ax1);
crossfit_open_athletes_2019_filtered.groupby(['division','category']).size().unstack().plot.barh(title="Ahtlete Division", ax=ax2, stacked=True, xticks=[100000,200000,300000]);
crossfit_open_athletes_2019_filtered.gender.value_counts().plot(kind='pie'
                                                                , autopct='%1.1f%%'
                                                                , pctdistance=0.5
                                                                , title='Ahtlete Gender'
                                                                , ax=ax3);
crossfit_open_athletes_2019_filtered.countryoforiginname.value_counts().plot(kind="pie"
                                 , autopct='%1.1f%%'
                                 , pctdistance=0.8
                                 , title='Ahtlete Country'
                                 , ax=ax4);
plt.tight_layout()
fig.subplots_adjust(wspace=0.5)
fig.subplots_adjust(hspace=0.5)
fig, (ax1, ax2, ax3) = plt.subplots(1,3,figsize=(20,10))
crossfit_open_athletes_2019_filtered.groupby(['category']).gender.value_counts().unstack().plot.bar(ax=ax1);
crossfit_open_athletes_2019_filtered.groupby(['category']).countryoforiginname.value_counts().unstack().plot.bar(ax=ax2,stacked=True);
crossfit_open_athletes_2019_filtered.groupby(['division']).countryoforiginname.value_counts().unstack().plot.bar(ax=ax3,stacked=True);
plt.tight_layout()
fig.subplots_adjust(wspace=0.5)
fig.subplots_adjust(hspace=0.5)
crossfit_open_athletes_2019_filtered[['height', 'weight', 'age']].describe()
height_lower_cap = crossfit_open_athletes_2019_filtered.height.quantile(0.02) # 1.5 m
height_upper_cap = crossfit_open_athletes_2019_filtered.height.quantile(0.99) # 1.96 m
weight_lower_cap = crossfit_open_athletes_2019_filtered.weight.quantile(0.01) # 51 kg
weight_upper_cap = crossfit_open_athletes_2019_filtered.weight.quantile(0.99) # 120.2 kg
age_lower_cap = crossfit_open_athletes_2019_filtered.age.min() # 16 years
#crossfit_open_athletes_2019_filtered.loc[crossfit_open_athletes_2019_filtered.age>=90].count()
age_upper_cap = 90
def hammond_ideal_weight_funct(height:list, height_units:str="cm"):
    if height_units == "cm":
        return 48 + 1.1 * (100*np.array(height) - 150)
    else:
        raise NotImplementedError
        
crossfit_athletes_height = np.array(crossfit_open_athletes_2019_filtered[~crossfit_open_athletes_2019_filtered.height.isna()].height.values.tolist())
ideal_open_athletes_weight = hammond_ideal_weight_funct(crossfit_athletes_height)
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2,2,figsize=(15,15))

crossfit_open_athletes_2019_filtered.height.plot.hist(bins = 10, range=(height_lower_cap, height_upper_cap), ax=ax1);

crossfit_open_athletes_2019_filtered.weight.plot.hist(bins = 10,range=(weight_lower_cap, weight_upper_cap), ax=ax2);

crossfit_open_athletes_2019_filtered.age.plot.hist(bins = 10,range=(age_lower_cap, age_upper_cap), ax=ax3);

crossfit_open_athletes_2019_filtered.plot.scatter(x='height'
                                                  , y='weight'
                                                  , xlim=(height_lower_cap, height_upper_cap)
                                                  , ylim=(weight_lower_cap, weight_upper_cap)
                                                  , ax=ax4
                                                  )

ax4.plot(crossfit_athletes_height
         , ideal_open_athletes_weight, color='black'
         , label='Hammond Ideal Weight')
ax4.legend()
plt.tight_layout()
fig.subplots_adjust(wspace=0.5)
fig.subplots_adjust(hspace=0.5)
crossfit_open_athletes_2019_filtered.groupby(['category','gender'])[['height','weight','age']].mean()
crossfit_open_athletes_2019_filtered.groupby(['division'])[['height','weight']].mean()
crossfit_open_athletes_2019_filtered.groupby(['countryoforiginname','gender'])[['height','weight','age']].mean()
crossfit_open_athletes_2019_filtered.isna().mean()
crossfit_open_athletes_2019_filtered[crossfit_open_athletes_2019_filtered.height.isna()].weight.isna().mean()<jupyter_output><empty_output><jupyter_text>Crossfit Feature Engineering<jupyter_code>from sklearn.preprocessing import StandardScaler

crossfit_open_athletes_2019_attribs = crossfit_open_athletes_2019_filtered \
                                    .loc[crossfit_open_athletes_2019_filtered.category != 'Scaled'] \
                                    .drop(['category','affiliateid','countryoforiginname','overallscore'],axis=1)

crossfit_open_athletes_2019_attribs \
    .loc[~crossfit_open_athletes_2019_attribs.height.between(height_lower_cap,height_upper_cap),"height"]= np.NaN

crossfit_open_athletes_2019_attribs \
    .loc[~crossfit_open_athletes_2019_attribs.weight.between(weight_lower_cap,weight_upper_cap),"weight"]= np.NaN

crossfit_open_athletes_2019_attribs['gender'].replace({'F':0,'M':1},inplace=True)
crossfit_open_athletes_2019_attribs['gender'] = crossfit_open_athletes_2019_attribs['gender'].astype('bool')
labels_binning = ['top_10','top_100','top_1000','top_10000','top_100000','top_200000']
no_bins = [0,10,100,1000,10000,100000,200000]
crossfit_open_athletes_2019_attribs['open_rank'] = pd.cut(crossfit_open_athletes_2019_attribs['overallrank'], bins=no_bins, labels=labels_binning)
crossfit_open_athletes_2019_attribs = pd.get_dummies(crossfit_open_athletes_2019_attribs, columns=['open_rank'])
crossfit_open_athletes_2019_attribs.fillna(crossfit_open_athletes_2019_attribs.mean(),inplace=True)
scaler = StandardScaler()
crossfit_open_athletes_2019_attribs[['age', 'height','weight']] = StandardScaler().fit_transform(crossfit_open_athletes_2019_attribs[['age', 'height','weight']])
####
# KERNEL DIES WHEN TRYING TO USE KNN IMPUTER FOR MISSING NAN VALUES 
# from sklearn.impute import KNNImputer
# imputer = KNNImputer(n_neighbors=3)
# crossfit_open_athlete_attribs_temp = imputer.fit_transform(crossfit_open_athlete_attribs_temp)
crossfit_open_athletes_2019_attribs.head()
crossfit_open_scores_2019 = pd.read_csv(crossfit_open_scores_2019_file)
crossfit_open_scores_2019.head()
crossfit_open_scores_2019_attribs = crossfit_open_scores_2019.copy()
labels_binning = ['top_10','top_100','top_1000','top_10000','top_100000','top_200000']
no_bins = [0,10,100,1000,10000,100000,200000]
crossfit_open_scores_2019_attribs['wod_rank'] = pd.cut(crossfit_open_scores_2019_attribs['rank'], bins=no_bins, labels=labels_binning)
crossfit_open_scores_2019_attribs = crossfit_open_scores_2019_attribs[['competitorid','wod_rank','division']]
crossfit_open_scores_2019_attribs = pd.get_dummies(crossfit_open_scores_2019_attribs, columns=['wod_rank'])
crossfit_open_scores_2019_attribs = crossfit_open_scores_2019_attribs.groupby(['competitorid','division']).sum().reset_index()
crossfit_open_athletes_scores_2019_attribs = crossfit_open_athletes_2019_attribs.merge(crossfit_open_scores_2019_attribs, on=['competitorid','division'], how='inner')
crossfit_open_athletes_scores_2019_attribs.head()
crossfit_games_athletes_scores_2019_attribs = pd.read_csv(crossfit_games_athletes_2019_file)
crossfit_games_athletes_scores_2019_attribs.head()
crossfit_games_athletes_scores_2019_attribs['games_rank'] = crossfit_games_athletes_scores_2019_attribs['overallrank'].apply(pd.to_numeric, errors='coerce')
crossfit_games_athletes_scores_2019_attribs = crossfit_games_athletes_scores_2019_attribs[['competitorid','games_rank','division']]
labels_binning = ['top_10','top_100','top_1000']
no_bins = [0,10,100,1000]
crossfit_games_athletes_scores_2019_attribs['games_rank'] = pd.cut(crossfit_games_athletes_scores_2019_attribs['games_rank'], bins=no_bins, labels=labels_binning)

crossfit_games_athletes_scores_2019_attribs.head()
crossfit_2019_model_data = crossfit_open_athletes_scores_2019_attribs.merge(crossfit_games_athletes_scores_2019_attribs, on=['competitorid','division'], how='left')
crossfit_2019_model_data = pd.get_dummies(crossfit_2019_model_data, columns=['division'])
crossfit_2019_model_data['games_rank'] = crossfit_2019_model_data['games_rank'].cat.add_categories('Not_Qualified')
crossfit_2019_model_data['games_rank'].fillna('Not_Qualified', inplace =True)
crossfit_2019_model_data['games_rank'] = crossfit_2019_model_data["games_rank"].cat.codes
crossfit_2019_model_data['games_rank'].value_counts() ## HIGHLY UNBALANCED
from sklearn.utils import resample
# Separate majority and minority classes
model_data_majority = crossfit_2019_model_data[crossfit_2019_model_data.games_rank==3]
model_data_minority = crossfit_2019_model_data[crossfit_2019_model_data.games_rank!=3]
 
# Downsample majority class
model_data_majority_downsampled = resample(model_data_majority, 
                                 replace=False,    # sample without replacement
                                 n_samples=model_data_minority.shape[0],     # to match minority class
                                 random_state=123) # reproducible results
 
# Combine minority class with downsampled majority class
crossfit_2019_model_data_downsampled = pd.concat([model_data_majority_downsampled, model_data_minority])
 
# Display new class counts
crossfit_2019_model_data_downsampled.games_rank.value_counts()
crossfit_2019_model_data_downsampled.head()<jupyter_output><empty_output><jupyter_text>CrossFit Model<jupyter_code>x = crossfit_2019_model_data_downsampled.drop(['competitorid','overallrank','games_rank'],axis=1)
x.head()
y = crossfit_2019_model_data_downsampled['games_rank']
y.head()
assert x.shape[0] == len(y)
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x, y, random_state=0)
clf = GradientBoostingClassifier(random_state=0)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))
from sklearn.metrics import precision_score
precision_score(y_test, y_pred, average='macro')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10746580973021448, 'fraction_numerical': 0.05732008071145654, 'mean_word_length': 5.765419615773508, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '51815', 'license': 'MIT', 'n_tokens_mistral': 5247, 'n_tokens_neox': 4470, 'n_words': 536, 'path': '[DS Nanodegree] - Project 1.ipynb', 'repo_name': 'Pabsm94/DataScientist_Nanodegree_Udacity', 'size': '13381'}