<jupyter_start><jupyter_text>Course on webscraping*By <NAME>, <NAME>, <NAME>*<jupyter_code># Imports:
import requests                     # for issueing HTTP requests
from bs4 import BeautifulSoup       # for parsing and navigating HTML results
import time                         # for sleeping between multiple requests

# Not related to webscraping
from IPython.display import Image   # for viewing images in a python notebook<jupyter_output><empty_output><jupyter_text>Documentation:- [Requests.py](http://docs.python-requests.org)- [Beautifulsoup.py](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)  Request, headers, user-agent, parameters, sleeping:<jupyter_code># Retrieving home page of Statistics Netherlands:
r1 = requests.get('https://www.cbs.nl/en-gb')

#r1.headers['content-type']
print(r1.status_code, r1.headers['content-type'], r1.encoding)
#print(r1.headers)
#print(r1.text)
# Retrieving home page of Statistics Netherlands with user-agent string:
headers = {'user-agent': 'scrapingCourseBot'}
r2 = requests.get('https://www.cbs.nl/en-gb', headers=headers)

# Headers of the request:
print(r2.request.headers)

# Headers of the response:
#print(r2.headers)
# Issue a request with parameters:
pars = {'products': 2, 'years': 2}
r3 = requests.get('http://testing-ground.webscraping.pro/table-middle.html?', params=pars, headers=headers)        
print(r3.url)
#print(r3.text)
# In a loop, always add some idle time (time.sleep) to not overload server:
for products in range(1, 4):
    for years in range(1, 3):
        pars = {'products': products, 'years': years}
        r4 = requests.get('http://testing-ground.webscraping.pro/table-middle.html?', params=pars, headers=headers)
        print(r4.url, r4.status_code)
        time.sleep(1)
# We can also querry other types of files
r5 = requests.get("http://testing-ground.webscraping.pro/img/logo.png", headers=headers)
r5.text
# This is not just text.
r5.content
# The .content attribute returns a binary string, which can be interpreted by other functions.
Image(r5.content)
r6 = requests.get("https://www.cbs.nl/-/media/_excel/2021/34/tozo_reg_2021_juni.xls", headers=headers)
r6.text
# Note that we write the binary. If you are merely writinng text, save respons.text and omit the "b" option.
# Always be mindfull when you download and save files, especially in binary!
if r6.status_code == 200:
    with open("bestand_tozo.xls", "wb") as file:
        file.write(r6.content)
# This works only if you are running this locally, pandas is not installed in the binder machines
import pandas as pd
pd.read_excel("bestand_tozo.xls")<jupyter_output><empty_output><jupyter_text>Beautifulsoup with native syntax (find, find_all):<jupyter_code># Get page with requests:
r4 = requests.get('https://www.cbs.nl/en-gb', headers=headers)
print(r4.url, r4.status_code)
#print(r4.request.headers)
#print(r4.text)
# Using soup to parse the html page:
soup = BeautifulSoup(r4.text, 'lxml')              # use lxml, is faster and more relaxed in parsing 
# find returns the first element:
print(soup.find("h2"))
#print(soup.find("h2").text)
#print(soup.find("h3").text)
# find the first element which belongs to a class:
part = soup.find("a", class_="thumbnail")
print(part)
# find the first element with an id:
aside = soup.find("section", id="aside-main")
print(aside)
# You can use find on a find result:
print(part.find("h3").text)
print(soup.find("a", class_="thumbnail").find("h3").text)
# how to get a URL from an a tag:
print(part['href'])
# find_all returns a list:
print(soup.find_all("h2"))
print("")
print(soup.find_all("h3"))
# You can iterate through a list output of find_all:
# Get the URLS to all news articles of CBS using find_all and find:
articles = soup.find_all("a", class_='thumbnail')
for article in articles:
    link = article['href']   # we retrieve the attribute href of the a tag
    print(link)
# Follow the links and get all texts of the news articles:
articles = soup.find_all("a", class_='thumbnail')
links3 = []
for article in articles:
    links3.append(article['href'])

for link in links3:
    r = requests.get(link, headers=headers)
    #print(r.url)
    soup2 = BeautifulSoup(r.text, 'lxml')
    leadtext = soup2.find('section', class_='leadtext')
    if leadtext is None: continue
    print(leadtext.text)
    time.sleep(1) # in robots.txt CBS advises a delay of 1 second<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13107678328032712, 'fraction_numerical': 0.014993184915947297, 'mean_word_length': 4.538364779874214, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 5, 'lorem ipsum': 0, 'www.': 5, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '92213', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 1491, 'n_tokens_neox': 1449, 'n_words': 427, 'path': '20220218/1_3_Course_Webscraping.ipynb', 'repo_name': 'SNStatComp/CBSAcademyBD', 'size': '4429'}