<jupyter_start><jupyter_text>Scrape detailed information about an athleteThis file takes the list of athletes found in scrape_athlete_list.ipynb and pulls detailed information about their events.Each athlete comes in with the following information:* Name* ID* School* School ID* DistrictAfter processing in this file the additional informacion is added to their record* 9th grade PR* 10th grade PR* 11th grade PR* 12th grade PR* Graduation Year<jupyter_code>import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import os

#athlete_no = '6483989' # <NAME>
#athlete_no = '3013148' # <NAME>
athlete_no = '1671873' 
url = 'https://www.athletic.net/TrackAndField/Athlete.aspx?AID='+athlete_no+'#!/L0'

response = requests.get(url)
# TEST CASE
#athlete_no = '6483989' # <NAME>
#athlete_no = '3013148' # <NAME>
athlete_no = '1671873' 
url = 'https://www.athletic.net/TrackAndField/Athlete.aspx?AID='+athlete_no+'#!/L0'

response = requests.get(url)
chromedriver = "/usr/local/bin/chromedriver"

os.environ["webdriver.chrome.driver"] = chromedriver

driver = webdriver.Chrome(chromedriver)
driver.get("https://www.athletic.net")
driver.get("https://www.athletic.net/account/login/?ReturnUrl=%2Fdefault.aspx")

%load_ext dotenv
%dotenv
EMAIL = '<EMAIL>'
#PASSWORD = os.environ.get('PASSWORD')
PASSWORD = '<PASSWORD>'
username_form = driver.find_element_by_name("email")
username_form.send_keys(EMAIL) # enter email

username_form.send_keys(Keys.RETURN)
password_form=driver.find_element_by_name("password") # note another approach
password_form.send_keys(PASSWORD) # enter password
password_form.send_keys(Keys.RETURN)<jupyter_output>The dotenv extension is already loaded. To reload it, use:
  %reload_ext dotenv<jupyter_text>Using Pandas<jupyter_code>header = {
  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
  "X-Requested-With": "XMLHttpRequest"
}

r = requests.get(url, headers=header)

dfs = pd.read_html(r.text)
# TEST CASE
for df in dfs:
    if df[0][0] == '400 Meters':
        all_years = df
        
print(all_years)
dfs[2]<jupyter_output><empty_output><jupyter_text>Pull data for several athletesLoop over a list of athletes from a csv file and read their 400 Meters numbers<jupyter_code>header = {
  "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
  "X-Requested-With": "XMLHttpRequest"
}
# given a time string, return a float for the time in seconds
def cleantime(time_str):
    time_str = time_str.replace('PR','')
    time_str = time_str.replace('*','')
    time_str = time_str.replace('c','')
    time_str = time_str.replace('h','')

    time_str = time_str.strip()
    #print('time = ',time_str)
    if ':' in time_str:
        t = time_str.split(':')
        time = float(t[0])*60 + float(t[1])
    elif time_str.replace('.','').isdigit(): 
        time = float(time_str)
    else: time = np.nan
    #print(time)
    return(time)
#file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athletes.csv'
#file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athlete_list.csv'
#file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athlete_list_boy1600.csv'
#file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athlete_list_girlls 1600.csv'
file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athlete_list_girl400.csv'
#file = '/Users/dana/metis/git/Metis_Bootcamp/Luther_Project/athlete_list_boy400.csv'

athlete_list = pd.read_csv(file)
# Given a list of athletes, scrap their PR times in the 400Meter race
# and add to dataframe
all_data = athlete_list[['ID','Name','School_ID','School','District']]
all_data = all_data.set_index(['ID'])
all_data['9_PR']  = np.nan
all_data['10_PR'] = np.nan
all_data['11_PR'] = np.nan
all_data['12_PR'] = np.nan
all_data['Grad_Yr'] = 0
all_data.Grad_Yr = all_data.Grad_Yr.astype(int)
all_data.head()
all_data.shape
#all_data1 = all_data[0:2000]
all_data2 = all_data[2000:3000]
all_data3 = all_data[3000:5000]
all_data4 = all_data[5000:8000]
all_data5 = all_data[8000:10000]
print(all_data1.shape,all_data2.shape,all_data3.shape,all_data4.shape,all_data5.shape)
all_data = all_data1
all_data.head()
all_data = all_data4

Event = '400 Meters'
#Event = '1600 Meters'
n = 0
for ID, row in all_data.iterrows():
    ID_str = str(ID)
    url = 'https://www.athletic.net/TrackAndField/Athlete.aspx?AID='+ID_str+'#!/L0'
    r = requests.get(url, headers=header)
    dfs = pd.read_html(r.text)
    for df in dfs:
        if df[0][0] == Event:
            n += 1
            if n%100 == 0: print(n)

            df.drop(df.index[0:1],inplace=True)
            for index, row in df.iterrows():
                if 'Indoor' not in str(row[0]):
                    #print(f"grade {int(row[1])}, time {row[2]}")
                    #row[1]= str(row[1]).replace('-','')
                    g = str(row[1])
                    g = g.replace('-','')
                    g = g.replace('.0','')

                    #print(ID_str,g)
                    if (g == '9' or g == '10' or g == '11' or g == '12'): 
                        #grade = str(int(row[1]))
                        grade = g

                    else:
                        grade = '0'
                    time = cleantime(str(row[2]))
                    #print(grade,time)
                    all_data.at[ID,grade+"_PR"] = time
# find the graduation year
                    if g == '12': 
                        gy = str(row[0])
                        gy = gy.replace(' Outdoor','')
                        all_data.at[ID,'Grad_Yr'] = int(gy)
                        #print(int(gy))

all_data4 = all_data
all_data2.head()
all_data
all_data = pd.concat([all_data1,all_data2,all_data3,all_data4])
#all_data = pd.concat([all_data1,all_data2])
all_data.columns
#all_data.drop(['8_PR'], axis=1,inplace=True)
#all_data.drop(['7_PR'], axis=1,inplace=True)
all_data.drop(['0_PR'], axis=1,inplace=True)
all_data.dropna(inplace=True)
all_data.dropna().shape
all_data.to_csv("allDistrict_girl400.csv")
all_data.shape
all_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1437489995197695, 'fraction_numerical': 0.0400192092204258, 'mean_word_length': 3.570592538405267, 'pattern_counts': {'":': 4, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 5, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '326119', 'license': 'MIT', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2264, 'n_words': 446, 'path': '2-Luther_Project/code/.ipynb_checkpoints/Project_Luther_athletes-checkpoint.ipynb', 'repo_name': 'Stitchmaker/Metis_Bootcamp', 'size': '6292'}