<jupyter_start><jupyter_text>Data munging<jupyter_code>import pandas as pd
import numpy as np
import datetime as dt
import re
race_df = pd.read_csv("../scraping/fis_scrape/worldloppet_races.csv")

#how many racers
print(len(race_df))

#define index for get function
#print(race_df.race_description.str.split()[1])

new_race_df = race_df.copy()
new_race_df.head(5)

#break down formatting one at a time to troubleshoot any new errors
#race date NB *very* long step
new_race_df['date'] = pd.to_datetime(race_df.race_description.str.split().str.get(0).str.replace(".", "-"))
new_race_df['year'] = new_race_df.date.dt.year
new_race_df['month'] = new_race_df.date.dt.month
new_race_df['week'] = new_race_df.date.dt.weekofyear
print(race_df.race_name.unique())

# first, setup a host country tag (good for the compare to racer_nationality rationale)
# so, idea being that if I can breakdown by host country... should be able to get most popular races(by country)

# then, define the splitting distance for short/long races. (optional, really)
# will have to condense the list of 173 unique race_names (fortunately plenty of redundancy
# should have to setup 20 ifelses like (if contains china == CHN or america == USA)
# setup a function which generates a 'host country' based on 'race_name' string

def host_country_gen(r):
    if 'Ushuaia' in r:
        return "ARG"
    elif 'American' in r:
            return "USA"
    elif 'Kangaroo' in r:
            return "AUS"
    elif 'Bieg' in r:
            return "POL"
    elif 'Transjurassienne' in r:
            return "FRA"
    elif 'Dolomitenlauf' in r:
            return "AUT"
    elif 'Marcialonga' in r:
            return "ITA"
    elif 'Engadin' in r:
            return "SUI"
    elif 'Padesatka' in r:
            return "CZE"
    elif 'Ludwig' in r:
            return "GER"
    elif 'Tartu' in r:
            return "EST"
    elif 'Finland' in r:
            return "FIN"
    elif 'China' in r:
            return "CHN"
    elif 'Sapporo' in r:
            return "JPN"
    elif 'Gatineau' in r:
            return "CAN"
    elif 'Birkebeinerrennet' in r:
            return "NOR"
    elif 'Fossavatnsgangan' in r:
            return "ISL"
    elif 'Merino' in r:
            return "NZL"
    elif 'Demino' in r:
            return "RUS"
    elif 'Vasaloppet' in r:
            return "SWE"
    else:
        return "none"
new_race_df['host_country'] = race_df.race_name.apply(host_country_gen)
print(new_race_df.host_country.unique())
len(new_race_df.host_country.unique())
# all countries accounted for !!  
# confirm no misattributes but with vasaloppet last should be OK
# setup quick numerical boolean to query tourist %'s
# 0 = racer is native; 1 = racer is a visitor (tourist)
new_race_df['tourist'] = np.where(new_race_df.racer_nationality != new_race_df.host_country, 1, 0)
#fastcheck
new_race_df.groupby('host_country').mean()['tourist'].sort_values(ascending = False)
#race distance
new_race_df['distance'] = pd.to_numeric(race_df.race_description.str.split().str.get(1).str.replace(',', '.'))
#race style
new_race_df['style'] = race_df.race_description.str.split().str.get(3)
new_race_df['style'].unique()
#racer nationality
new_race_df['racer_nationality'] = new_race_df['racer_nationality'].str.upper()
#racer time 
new_race_df['racer_time'] = pd.to_timedelta(new_race_df.racer_time.str.extract('(\d+[:|.]\d{2}[:|.]\d{2})').str.replace(".", ":"))
new_race_df['racer_pace'] = new_race_df['distance'] / (new_race_df['racer_time'] / dt.timedelta(hours=1))
### after evaluating racer_pace, I came to the realization that there are some "DNF" finish times
### therefore, I will split the df into two: race_finishers and race_dnf using null/notnull arguments

race_finishers = new_race_df[pd.notnull(new_race_df.racer_pace)].copy()

race_dnf = new_race_df[pd.isnull(new_race_df.racer_pace)].copy()
race_finishers.head()
#get the number of skiers on the basis of how many rows for a given race_description
race_finishers['n_skiers'] = race_finishers['race_name'].groupby(race_finishers['race_description']).transform('count')<jupyter_output><empty_output><jupyter_text>Preliminary data visualization / EDA<jupyter_code>%matplotlib inline
from matplotlib import pyplot as plt
plt.style.use('ggplot')
race_finishers.groupby('host_country')['tourist'].mean().sort_values().plot(kind= 'barh', color = 'b')
plt.ylabel('Host Country')
plt.xlabel('Tourist skiiers (%)')
race_finishers.groupby('host_country')['tourist'].sum().sort_values().plot(kind= 'barh', color = 'b')
plt.ylabel('Host Country')
plt.xlabel('Tourist skiiers (n)')
race_finishers.groupby('host_country')['racer_pace'].median().sort_values().plot(kind = 'barh', color='b')
plt.xlabel('race pace (km/h)')
plt.ylabel('Host Country')
plt.title('Median race pace by host country', fontsize=20)
# consider a split for long/short races? some division by distance
race_finishers.groupby(['host_country', 'year', 'distance', 'style'])['racer_pace'].median().sort_values().plot(kind = 'barh', color='b')
plt.xlabel('race pace (km/h)')
plt.ylabel('Host Country')
plt.title('Median race pace by race', fontsize=20)
race_finishers.groupby(['host_country', 'year', 'distance', 'style'])['racer_pace'].median().sort_values(ascending = False).tail(25)
len(race_finishers.groupby(['host_country', 'year', 'distance', 'style'])['racer_pace'].median())
plt.hist((race_finishers.racer_pace), bins = 30)
plt.xlabel('race pace (km/h)')
plt.ylabel('count')
plt.title('Race pace distribution', fontsize=20)
plt.hist(race_finishers.groupby('race_name')['distance'].mean(), bins = 10)
#this is a rough pass on the distance metric but looks like I need to apply normalization (box-cox?)
#number of racers at a given race
plt.hist(race_finishers.groupby(['host_country', 'year', 'distance', 'style'])['month'].count(), bins = 15)
#highly skewed ... log transform or something
compiled_table = race_finishers.groupby(['host_country', 'year', 'distance', 'style', 'month', 'week']).agg({'racer_pace' : ['median', 'max'], 'year' : ['count'], 'tourist' : ['mean']}).reset_index()
#df.reset_index(level='class', col_level=1, col_fill='species')
compiled_table
compiled_table.columns
compiled_table.columns = compiled_table.columns.get_level_values(0)
print(compiled_table.columns)

compiled_table.columns.values[6] = 'median_pace'
compiled_table.columns.values[7] = 'max_pace'
compiled_table.columns.values[8] = 'n_ski'
print(compiled_table.columns)

#backup as checkpoint
compiled_table.to_csv("worldloppet_munged.csv")
#split finishers into classic or skate 
classic = compiled_table[compiled_table['style']=='CT'].copy()
print("number of classic races: " + str(len(classic)))
skate = compiled_table[compiled_table['style']=='FT'].copy()
print("number of skate races: " + str(len(skate)))<jupyter_output>number of classic races: 186
number of skate races: 145<jupyter_text>Prep for KNN classification<jupyter_code>race_finishers = race_finishers[race_finishers['racer_pace']>1]
print(race_finishers.head(2))

#define distance bounds
print(race_finishers.distance.min())
print(race_finishers.distance.max())

#define pace bounds
print(race_finishers.racer_pace.min())
print(race_finishers.racer_pace.max()) # yeah right, olympic bounds are like 27...

#define race size bounds
print(race_finishers.n_skiers.min())
print(race_finishers.n_skiers.max()) # yeah right, olympic bounds are like 27...

race_finishers.groupby(['host_country', 'year', 'distance', 'style'])['month'].count()<jupyter_output><empty_output><jupyter_text>Determining lambda value for box-cox transformation<jupyter_code>#normalize data prior to KNN w/ box-cox transformation
from scipy import stats

bc_dist, maxlog, interval = stats.boxcox(race_finishers.distance, alpha=0.05)
print("dist_lambda = {:g}".format(maxlog))

bc_pace, maxlog, interval = stats.boxcox(race_finishers.racer_pace, alpha=0.05)
print("pace_lambda = {:g}".format(maxlog))

bc_nski, maxlog, interval = stats.boxcox(race_finishers.n_skiers, alpha=0.05)
print("nski_lambda = {:g}".format(maxlog))

race_finishers['bc_dist'] = bc_dist
race_finishers['bc_pace'] = bc_pace
race_finishers['bc_nski'] = bc_nski

race_finishers.head(2)
# filter by ski-style as these are different races
import math
classic = race_finishers[race_finishers['style']=='CT'].copy()
print("number of classic racers: " + str(len(classic)))
print("sqrt(n) for K-neighbors, classic: " + str(math.sqrt(len(classic))))
skate = race_finishers[race_finishers['style']=='FT'].copy()
print("number of skate racers: " + str(len(skate)))
print("sqrt(n) for K-neighbors, skate: " + str(math.sqrt(len(skate))))
#finalized output for training model
model_ski_df = race_finishers.iloc[:,[11,8,14,15,16,6]]

print(model_ski_df.head(5))
model_ski_df.to_csv("model_ski_df.csv")<jupyter_output>style host_country   bc_dist   bc_pace    bc_nski  month
0    CT          ARG  8.775961  2.471600  14.721642     12
1    CT          ARG  8.775961  2.388236  14.721642     12
2    CT          ARG  8.775961  2.332943  14.721642     12
3    CT          ARG  8.775961  2.328083  14.721642     12
4    CT          ARG  8.775961  2.322862  14.721642     12<jupyter_text>KNN modeling for classic racers<jupyter_code>import numpy as np
from sklearn.cross_validation import train_test_split

# create attribute matrix (bc_dist, bc_pace) and target (host_country) 
ski_data = np.array(classic.iloc[:,[13,14]]) 
host = np.array(classic.host_country) 

# setup train and test sets
ski_data_train, ski_data_test, host_train, host_test = train_test_split(ski_data, host, test_size=0.20, random_state=9)
#now, test k iteratively starting with classic
#note, sqrt(n) printouts above

from sklearn.model_selection import cross_val_score

k_neighbors = [n for n in range(1,1000, 100) if n % 2 != 0] #start in 100s & avoid ties using odd neighbors

ct_cv_scores = []

# for i in k_neighbors:
#     knn = KNeighborsClassifier(n_neighbors=i)
#     scores = cross_val_score(knn, ski_data_train, host_train, cv=10, scoring='accuracy', n_jobs = -1)
#     ct_cv_scores.append(scores.mean())
    
# changing to misclassification error
MSE = [1 - x for x in ct_cv_scores]

# determining best k
optimal_k = k_neighbors[MSE.index(min(MSE))]
print ("The optimal number of neighbors is %d" % optimal_k)

# plot misclassification error vs k
plt.plot(k_neighbors, MSE)
plt.xlabel('Number of Neighbors K')
plt.ylabel('Misclassification Error')
plt.show()<jupyter_output>The optimal number of neighbors is 101<jupyter_text>interpretation:Given the sqrt(n) rule of thumb, I assumed that the optimal K neibhbors would be closer to 601. Turns out, the minimum error from this computational run is between 1 and 201. Let's repeat between 1 and 201 to generate a K to use.<jupyter_code>k_neighbors_2 = [n for n in range(1,200, 10) if n % 2 != 0] #start in 100s & avoid ties using odd neighbors

ct_cv_scores_2 = []

# for i in k_neighbors_2:
#     knn = KNeighborsClassifier(n_neighbors=i)
#     scores = cross_val_score(knn, ski_data_train, host_train, cv=10, scoring='accuracy', n_jobs = -1)
#     ct_cv_scores_2.append(scores.mean())
    
# changing to misclassification error
MSE_2 = [1 - x for x in ct_cv_scores_2]

# determining best k
optimal_k_2 = k_neighbors_2[MSE_2.index(min(MSE_2))]
print ("The optimal number of neighbors is %d" % optimal_k_2)

# plot misclassification error vs k
plt.plot(k_neighbors_2, MSE_2)
plt.xlabel('Number of Neighbors K')
plt.ylabel('Misclassification Error')
plt.show()<jupyter_output>The optimal number of neighbors is 71<jupyter_text>KNN modeling for skate racers<jupyter_code>import numpy as np
from sklearn.cross_validation import train_test_split

# create attribute matrix (bc_dist, bc_pace) and target (host_country) 
skate_ski_data = np.array(skate.iloc[:,[13,14]]) 
skate_host = np.array(skate.host_country) 

# setup train and test sets
ski_data_train2, ski_data_test2, host_train2, host_test2 = train_test_split(skate_ski_data, skate_host, test_size=0.20, random_state=9)
#repeat process as before

from sklearn.model_selection import cross_val_score

k_neighbors = [n for n in range(1,1000, 100) if n % 2 != 0] #start in 100s & avoid ties using odd neighbors

ft_cv_scores = []

for i in k_neighbors:
    knn = KNeighborsClassifier(n_neighbors=i)
    scores = cross_val_score(knn, ski_data_train2, host_train2, cv=10, scoring='accuracy', n_jobs = -1)
    ft_cv_scores.append(scores.mean())
# changing to misclassification error
MSE3 = [1 - x for x in ft_cv_scores]

# determining best k
optimal_k = k_neighbors[MSE3.index(min(MSE3))]
print ("The optimal number of neighbors is %d" % optimal_k)

# plot misclassification error vs k
plt.plot(k_neighbors, MSE3)
plt.xlabel('Number of Neighbors K')
plt.ylabel('Misclassification Error')
plt.show()
k_neighbors_2 = [n for n in range(1,200, 10) if n % 2 != 0] #start in 100s & avoid ties using odd neighbors

ft_cv_scores_2 = []

for i in k_neighbors_2:
    knn = KNeighborsClassifier(n_neighbors=i)
    scores = cross_val_score(knn, ski_data_train2, host_train2, cv=10, scoring='accuracy', n_jobs = -1)
    ft_cv_scores_2.append(scores.mean())
# changing to misclassification error
MSE_4 = [1 - x for x in ft_cv_scores_2]

# determining best k
optimal_k_2 = k_neighbors_2[MSE_4.index(min(MSE_4))]
print ("The optimal number of neighbors is %d" % optimal_k_2)

# plot misclassification error vs k
plt.plot(k_neighbors_2, MSE_4)
plt.xlabel('Number of Neighbors K')
plt.ylabel('Misclassification Error')
plt.show()<jupyter_output>The optimal number of neighbors is 51
{'fraction_non_alphanumeric': 0.1281747500925583, 'fraction_numerical': 0.024435394298407995, 'mean_word_length': 5.1727605118829985, 'pattern_counts': {'":': 1, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '196196', 'license': 'MIT', 'n_tokens_mistral': 5189, 'n_tokens_neox': 4753, 'n_words': 1190, 'path': 'munging/ski_race_munging.ipynb', 'repo_name': 'mikeacaballero/ski-tourism', 'size': '13505'}