<jupyter_start><jupyter_text>Request html document from the webserver<jupyter_code>!pip install requests
import requests
url = 'https://www.lyrics.com/artist/Laurie-Anderson/3545'
response = requests.get(url)
# Check the status code
response.status_code

# 200: means that the request was valid and we got what we asked for
# 404: Page not found

# Everything with 400 means that we made a mistake or are not authorized

# Everything with 500 means that there is an error on the server side
# Where is the html
print(response.text[:40])
html = response.text<jupyter_output><empty_output><jupyter_text>How can we use the structure that the html document provides us to our advantage?-> Parse html<jupyter_code>!pip install beautifulsoup4
from bs4 import BeautifulSoup
parsed_html = BeautifulSoup(html)
# print(parsed_html.prettify())
# Find the body of the html document
parsed_html.find('body').find('div')<jupyter_output><empty_output><jupyter_text>Extract the curriculum from https://spiced.academy/de/program/data-science<jupyter_code># Request the page
spiced_response = requests.get('https://spiced.academy/de/program/data-science')
spiced_response.status_code
# Turn this into a BeautifulSoup object
parsed_spiced_html = BeautifulSoup(spiced_response.text)
#print(parsed_spiced_html.prettify())
# Find all h3 tags
parsed_spiced_html.find_all('h3', attrs={'class': 'mob-hidden'})[1].text

# returns a bs4.ResultSet; it behaves like a list
for entry in parsed_spiced_html.find_all('h3', attrs={'class': 'mob-hidden'})[1:]:
    print(entry.text)
    print('\n')
# You can walk down the hierarchies with beautiful soup
parsed_spiced_html.find('body').div.a.get('href')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10647058823529412, 'fraction_numerical': 0.015294117647058824, 'mean_word_length': 6.363636363636363, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '144883', 'license': 'MIT', 'n_tokens_mistral': 563, 'n_tokens_neox': 553, 'n_words': 155, 'path': 'web_scraping.ipynb', 'repo_name': 'fserro/lyrics-classifier', 'size': '1700'}