<jupyter_start><jupyter_text>Scratch Work<jupyter_code>soup = soup_maker(url)
scraped_data = []
tbl = soup.find('table',{'class':'table table-hover persist-area'})
tbdy = tbl.find('tbody')
all_a = tbdy.find_all('a', {'rel':None})
player_details = {}
# players on every other index
for i, lnk in enumerate(all_a):
    if i % 2 == 0:
        player_details['short_name'] = lnk.text
        #player_details.update(get_player_details('http://sofifa.com' + lnk['href']))
        player_details['link'] = 'http://sofifa.com' + lnk['href']
        print(player_details)
        scraped_data.append(player_details)
all_deets = {}
soup = soup_maker('http://sofifa.com/player/20801/c-ronaldo-dos-santos-aveiro/')
plyr_info = soup.find('div', {'class':'player'})

# find_plyr_info
plyr_dat = {} # prepare dictionary for player data
ply_inf = plyr_info.find('div', {'class':'meta'}) # grab section with player data
infos = ply_inf.text
data_str = infos[infos.index('Age') + 4:] # grab all data in str to right of Age like height and weight

# set variables
plyr_dat['pref_pos'] = ply_inf.find('span').text
plyr_dat['full_name'] = ply_inf.text[0:ply_inf.text.find(plyr_dat['pref_pos'])-2].strip()
plyr_dat['age'] = int(data_str[:2])
plyr_dat['height'] = data_str[data_str.index(')') + 2:].split(' ')[0].replace('\"','')
plyr_dat['weight'] = data_str[data_str.index(')') + 2:].split(' ')[1]
print(plyr_dat)
plyr_data = {}
soup = soup_maker('http://sofifa.com/player/20801/c-ronaldo-dos-santos-aveiro/')
plyr_stats = soup.find('div', {'class':'stats'})
plyr_val = plyr_stats.text[plyr_stats.text.find('€'):].split('\n')[0]
info = re.findall('\d+', plyr_stats.text)
plyr_data['rating'] = int(info[0])
plyr_data['potential'] = int(info[1])

if 'M' in plyr_val:
    plyr_data['value'] = int(plyr_val[1:plyr_val.index('M')])*1000000
elif 'K' in plyr_val:
    plyr_data['value'] = int(plyr_val[1:plyr_val.index('K')])*1000

plyr_data['wage'] = int(info[3])*1000

print(plyr_data)
soup = soup_maker('http://sofifa.com/player/20801/c-ronaldo-dos-santos-aveiro/')
sp = soup.find('div', {'class':'teams'})
lnks = sp.find_all('li')
club = lnks[8].find('a').text
joined = lnks[12].text
print(club)
print(lnks[12].find('label', text='Joined').parent.contents[2])<jupyter_output>Juventus
Jul 10, 2018<jupyter_text>Ok, now let's turn this work into functions<jupyter_code>def gather_basic_info(soup):
    plyr_dat = {} # prepare dictionary for player data
    ply_inf = soup.find('div', {'class':'meta'}) # grab section with player data
    infos = ply_inf.text
    if 'Age' not in infos:
        return(plyr_dat)
    data_str = infos[infos.index('Age') + 4:] # grab all data in str to right of Age like height and weight

    # set variables
    plyr_dat['pref_pos'] = ply_inf.find('span').text
    plyr_dat['full_name'] = ply_inf.text[0:ply_inf.text.find(plyr_dat['pref_pos'])-2].strip()
    plyr_dat['age'] = int(data_str[:2])
    plyr_dat['height'] = data_str[data_str.index(')') + 2:].split(' ')[0].replace('\"','')
    plyr_dat['weight'] = data_str[data_str.index(')') + 2:].split(' ')[1]
    return(plyr_dat)
def gather_club_info(soup):
    # gets club and join data info
    plyr_data = {}
    lnks = soup.find_all('li')
    
    # the link list isn't the same for each player
    if lnks[8].find('a'):
        plyr_data['Club'] = lnks[8].find('a').text
    else:
        plyr_data['Club'] = lnks[7].find('a').text
        
    # assign data
    if len(lnks) < 13:
        plyr_data['Joined'] = None
        return(plyr_data)
    
    if lnks[12].find('label', text='Joined'):
        plyr_data['Joined'] = lnks[12].find('label', text='Joined').parent.contents[2]
    else:
        plyr_data['Joined'] = None
    
    return(plyr_data)
def gather_player_stats(soup):
    # parse stats section and determine rating, potential, value, and wage
    plyr_data = {}
    plyr_val = soup.text[soup.text.find('€'):].split('\n')[0]
    info = re.findall('\d+', soup.text)
    
    plyr_data['rating'] = int(info[0])
    plyr_data['potential'] = int(info[1])

    # check units of their value
    if 'M' in plyr_val:
        plyr_data['value'] = int(float(plyr_val[1:plyr_val.index('M')])*1000000)
    elif 'K' in plyr_val:
        plyr_data['value'] = int(float(plyr_val[1:plyr_val.index('K')])*1000)

    # wage is always given in thousands
    if len(info) > 4:
        plyr_data['wage'] = int(info[4])*1000
    else:
        plyr_data['wage'] = int(info[3])*1000
    
    return(plyr_data)
def get_player_details(soup):
    # take in player url and build dictionary of player info
    all_deets = {}
    soup = soup_maker(soup)
    
    # gather basic name, height, weight stats
    plyr_info = soup.find('div', {'class': 'player'})
    all_deets.update(gather_basic_info(plyr_info))
    
    # gather rating, value, wage, etc
    plyr_stats = soup.find('div', {'class': 'stats'})
    all_deets.update(gather_player_stats(plyr_stats))
    
    club_info = soup.find('div', {'class': 'teams'})
    all_deets.update(gather_club_info(club_info))
    
    return(all_deets)<jupyter_output><empty_output><jupyter_text>Now let's create our dataframe<jupyter_code>def scrape_sofifa():
    # completes entire scraping process, can take minutes to run
    scraped_data = []
    page_indexes = [i*60 for i in range(100)]
    
    # iterate across 100 different player pages
    for page_index in page_indexes:
        print('Scraping page {}...'.format(page_index))
        url = 'https://sofifa.com/players?offset=' + str(page_index)
        soup = soup_maker(url)
        
        # gather all player details for this page
        tbl = soup.find('table',{'class':'table table-hover persist-area'})
        tbdy = tbl.find('tbody')
        all_a = tbdy.find_all('a', {'rel':None})
        
        # players are on every other index
        for i, lnk in enumerate(all_a):
            if i % 2 == 0:
                player_details = {}
                player_details['short_name'] = lnk.text
                player_details['link'] = 'http://sofifa.com' + lnk['href']
                player_details.update(get_player_details('http://sofifa.com' + lnk['href']))
                scraped_data.append(player_details)    
    
    return(scraped_data)
import pandas as pd
scraped_data = scrape_sofifa()
df = pd.DataFrame(scraped_data)
df.head()
df.to_csv('scraped_sofifa.csv', index=False)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17066290550070523, 'fraction_numerical': 0.017708823068484564, 'mean_word_length': 3.802106847253574, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '514195', 'license': 'MIT', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2368, 'n_words': 436, 'path': 'sofifa_scrape.ipynb', 'repo_name': 'raghavagovil/FIFA-Data-Analysis', 'size': '6381'}