<jupyter_start><jupyter_text>Machine learning on rides of 2017  Preparation and function definitions  Function to import a CSV with grain data of a ride<jupyter_code># %load training_functions.py
import pandas as pd
import os
import numpy as np
from datetime import datetime

def read_csv_power_file(file_path, filename):
    csv_path = os.path.join(file_path, filename)
    df = pd.read_csv(csv_path)
    
    # Drop columns that we don't need
    df.drop('lat', axis=1, inplace=True)
    df.drop('lon', axis=1, inplace=True)
    df.drop('nm', axis=1, inplace=True)
    df.drop('hhb', axis=1, inplace=True)
    df.drop('o2hb', axis=1, inplace=True)
    df.drop('thb', axis=1, inplace=True)
    df.drop('smo2', axis=1, inplace=True)
    df.drop('rps', axis=1, inplace=True)
    df.drop('lps', axis=1, inplace=True)
    df.drop('rte', axis=1, inplace=True)
    df.drop('lte', axis=1, inplace=True)
    df.drop('headwind', axis=1, inplace=True)
    df.drop('slope', axis=1, inplace=True)
    
    # Replace 0's in columns like cadence
    df['cad'].replace(0, value=np.NaN, inplace=True)
    df['kph'].replace(0, value=np.NaN, inplace=True)
    df['hr'].replace(0, value=np.NaN, inplace=True)
    
    return df


def calculate_height_gain(hd):
    old = hd[0]
    height_gain = 0
    
    for v in hd:
        if v > old:
            height_gain += v-old
        old = v
    
    return height_gain

def normalized_power(pd):
    return np.power(pd.pow(4).sum() / len(pd), 0.25)

def hr_drift(hrd, pd):
    l = len(hrd)/2
    
    first_half = hrd[:l].mean() / pd[:l].mean()
    second_half = hrd[l:].mean() / pd[l:].mean()
    
    return 1 - (first_half / second_half)

def kilojoules(pd):
    return (pd.mean()*len(pd))/1000

def do_aggregations(d, epoch_day, filename):
    df = pd.DataFrame(columns=('filename', 'epoch_day', 'time', 'cad', 'hr', 'hr_min', 'hr_max', 'hr_drift', 
                               'km', 'kph', 'kilojoules', 'watts', 'watts_max', 'watts_std', 'watts_25', 
                               'watts_50', 'watts_75', 'np', 'alt', 'temp', 'lrbalance', 'if'))
    
    alt = calculate_height_gain(d['alt'].values)
    hrdrift = hr_drift(d['hr'], d['watts'])
    
    df.loc[0] = [filename, epoch_day, len(d['hr'])/60, d['cad'].mean(), d['hr'].mean(),d['hr'].min(), 
                 d['hr'].max(), hrdrift, d['km'].max(), d['kph'].mean(), kilojoules(d['watts']), 
                 d['watts'].mean(), d['watts'].max(), d['watts'].std(), d['watts'].quantile(q=0.25), 
                 d['watts'].quantile(q=0.50), d['watts'].quantile(q=0.75), normalized_power(d['watts']), 
                 alt, d['temp'].mean(), d['lrbalance'].mean(), normalized_power(d['watts'])/d['watts'].mean()]
    
    df.filename.apply(str)
    
    df['kph'].fillna(0, inplace=True)
    
    return df

def get_epoch_day(date_s):
    # 2017_06_05_19_59_30.csv
    day = date_s.split('.')[0]
    return (datetime.strptime(day, '%Y_%m_%d_%H_%M_%S') - datetime(1970,1,1)).days<jupyter_output><empty_output><jupyter_text>Read power data generating a dataframe called 't_data'<jupyter_code>from os import listdir
from os.path import isfile, join

def files_in_folder(mypath):
    return [f for f in listdir(mypath) if isfile(join(mypath, f))]

path = '/home/caster/training_data'

t_data = pd.concat([do_aggregations(read_csv_power_file(path, filename), get_epoch_day(filename), filename) for filename in files_in_folder(path) if filename.split('.')[1] == 'csv'])
t_data.sort_values('epoch_day', axis=0, inplace=True)
t_data[:10]<jupyter_output>/usr/lib/python2.7/site-packages/ipykernel/__main__.py:51: RuntimeWarning: divide by zero encountered in double_scalars
/usr/lib/python2.7/site-packages/ipykernel/__main__.py:52: RuntimeWarning: divide by zero encountered in double_scalars
/usr/lib/python2.7/site-packages/ipykernel/__main__.py:54: RuntimeWarning: invalid value encountered in double_scalars
/usr/lib/python2.7/site-packages/ipykernel/__main__.py:71: RuntimeWarning: invalid value encountered in double_scalars<jupyter_text>Linear regression HR with Power data (using Keras)<jupyter_code>import matplotlib.pyplot as plt

from sklearn.preprocessing import StandardScaler

import keras
from keras.layers import Input, Dense
from keras.models import Model

import numpy as np

# Normalize
sc = StandardScaler()

qw = t_data["watts"].quantile(0.02)
cur = t_data[t_data['watts'] > qw]

x_ = cur['hr']
y_ = cur['watts']

x = x_
x = sc.fit_transform(x_.values.reshape(-1, 1))
y = y_
# y = sc.fit_transform(y_.values.reshape(-1, 1))

# Model
inputs = Input(shape=(1,))
preds = Dense(1, activation='linear')(inputs)

model = Model(inputs=inputs,outputs=preds)
sgd = keras.optimizers.SGD()

model.compile(optimizer=sgd ,loss='mse',metrics=['mse'])
model.fit(x,y, batch_size=1, epochs=30, shuffle=False, verbose=0)

# Plot
plt.figure()
plt.scatter(x_,y, color='black')

plt.xlabel('HR')
plt.ylabel('Watts')

plt.plot(x_, model.predict(x), color='blue', linewidth=1)

plt.show()

slope = model.get_weights()[0].item(0)
interceptor = model.get_weights()[1].item(0)

print("Slope:       %f" % slope)
print("Interceptor: %f" % interceptor)
pr_hr = 160

norm_hr = sc.transform(np.array([pr_hr]).reshape(-1, 1))
a = model.predict(norm_hr).item(0)

print("%f (%f) -> %f watts" % (pr_hr, norm_hr, a))<jupyter_output>160.000000 (1.597836) -> 179.983093 watts<jupyter_text>Clustering over most features<jupyter_code>from sklearn import cluster
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
qw = t_data["watts"].quantile(0.03)
cur = t_data[t_data['watts'] > qw]

k_means = cluster.KMeans(n_clusters=2)
k_means.fit(cur.drop(['filename', 'kph', 'watts_75', 'watts_25', 'watts_50', 'lrbalance'], axis=1))

cur = cur.assign(c=k_means.labels_)

plt.scatter(x_, y_, c=cur['c'])

plt.xlabel('HR')
plt.ylabel('Watts')

plt.show()
cur[cur['c'] == 1].describe().drop(['count']).drop(['kph', 'lrbalance', 'c'], axis=1)
cur[cur['c'] == 0].describe().drop(['count']).drop(['kph', 'lrbalance', 'c'], axis=1)<jupyter_output><empty_output><jupyter_text>It seems that the 'differentiating' feature is kilojoules. We can see that the mean of kilojoules is:* Group 1: 645 kj* Group 2: 1205 kjWhile the rest of the data has similar characteristics  Regression tree<jupyter_code># MLP for Pima Indians Dataset with 10-fold cross validation via sklearn
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
import numpy
from sklearn.ensemble import RandomForestClassifier

from sklearn.tree import DecisionTreeRegressor

# Normalize
sc = StandardScaler()

qw = t_data["watts"].quantile(0.02)
cur = t_data[t_data['watts'] > qw]

x_ = cur['hr']
y_ = cur['watts']

X = x_[:, None]
# X = sc.fit_transform(x_.values.reshape(-1, 1))
y = y_
# y = sc.fit_transform(y_.values.reshape(-1, 1))


# Fit regression model
regr_1 = DecisionTreeRegressor(max_depth=2)
regr_2 = DecisionTreeRegressor(max_depth=3)
regr_3 = DecisionTreeRegressor(max_depth=5)
regr_1.fit(X, y)
regr_2.fit(X, y)
regr_3.fit(X, y)

# Predict
X_test = np.arange(x_.min(), x_.max(), 1)[:, np.newaxis]
y_1 = regr_1.predict(X_test)
y_2 = regr_2.predict(X_test)
y_3 = regr_3.predict(X_test)

# Plot the results
plt.figure(figsize=(15, 6), dpi=80)
plt.scatter(X, y, c="darkorange", label="data")
plt.plot(X_test, y_1, color="blue", label="max_depth=2", linewidth=1)
plt.plot(X_test, y_2, color="red", label="max_depth=3", linewidth=1)
plt.plot(X_test, y_3, color="green", label="max_depth=5", linewidth=1)
plt.xlabel("heart rate")
plt.ylabel("watts")
plt.title("Decision Tree Regression")
plt.legend()
plt.show()

pr_hr = 155
p1 = regr_1.predict(pr_hr)
p2 = regr_2.predict(pr_hr)
p3 = regr_3.predict(pr_hr)


print("%f bpm Depth 2: %f watts" % (pr_hr, p1))
print("%f bpm Depth 3: %f watts" % (pr_hr, p2))
print("%f bpm Depth 5: %f watts" % (pr_hr, p3))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17193650793650794, 'fraction_numerical': 0.02819047619047619, 'mean_word_length': 4.73216885007278, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '610431', 'license': 'Apache-2.0', 'n_tokens_mistral': 3207, 'n_tokens_neox': 3051, 'n_words': 620, 'path': 'Rides (CSV version).ipynb', 'repo_name': 'sayden/machine-learning-on-cycling-data', 'size': '7875'}