<jupyter_start><jupyter_text>Web scraping 101After https://fcpython.com/scraping/introduction-scraping-data-transfermarkt<jupyter_code>import requests
from bs4 import BeautifulSoup
import pandas as pd

# Installed requests from conda, bs4 via pip
headers = {'User-Agent': 
           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'}

page = "https://www.transfermarkt.co.uk/transfers/transferrekorde/statistik/top/plus/0/galerie/0?saison_id=2000"
pageTree = requests.get(page, headers=headers)
pageSoup = BeautifulSoup(pageTree.content, 'html.parser')
Players = pageSoup.find_all("a", {"class": "spielprofil_tooltip"})

#Let's look at the first name in the Players list.

Players[0].text
Values = pageSoup.find_all("td", {"class": "rechts hauptlink"})

Values[0].text
PlayersList = []
ValuesList = []

for i in range(0,25):
    PlayersList.append(Players[i].text)
    ValuesList.append(Values[i].text)
    
df = pd.DataFrame({"Players":PlayersList,"Values":ValuesList})

df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14392523364485982, 'fraction_numerical': 0.041121495327102804, 'mean_word_length': 5.611111111111111, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '637159', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 419, 'n_tokens_neox': 383, 'n_words': 72, 'path': 'fcpython/Scraping 101.ipynb', 'repo_name': 'syphax/CrounseCode', 'size': '1070'}