<jupyter_start><jupyter_text>Start by downloading <NAME>'s page<jupyter_code># url = 'https://en.wikipedia.org/wiki/George_Washington'

# response = requests.get(url)
# response.status_code  #200 = success!
# assert response.status_code == 200
# response.text[:1000]  #First 1000 characters of the HTML
# page = response.text
# soup = BeautifulSoup(page, 'html5lib')
# print(soup.prettify()[:1000])
# [object2.find('p') for object2 in [object.find_all('div', recursive=False) for object in soup.find_all('body')]]
# divs = soup.find('body').find_all('div', recursive=False)
# divs
# len(page)<jupyter_output><empty_output><jupyter_text>Now, try to download and get a list of websites of presidencies of the united states.<jupyter_code>url2 = 'https://en.wikipedia.org/wiki/List_of_presidents_of_the_United_States'

response = requests.get(url2)
assert response.status_code == 200, f'server recorded on error on reguest {url2}'
soup = BeautifulSoup(response.text)
def parse_presidential_links(soup):
    """Takes the soup object for the website 
    <list of presidents of united states>, and 
    returns a list of links to their wikipedia pages
    Note: The links are relative to the wikipedia homepage
    """
    
    rows = (soup.find(class_='wikitable').find('tbody')
               .find_all('tr'))
    links = []
    for row in rows: 
        if len(row.find_all('td')) > 0:
            if row.find_all('td')[0].find('a') is not None:
                a_tag = row.find_all('td')[0].find('a')
                if a_tag.get('title')[0:13] == 'Presidency of':
                    links.append(a_tag.get('href'))
    return links


        
        
    
links = parse_presidential_links(soup)
print(len(links))
print(links)

import datetime
soups = []
for link in links:
    wiki_url = 'https://en.wikipedia.org/'
    full_url = wiki_url + link
    response = requests.get(full_url)
    assert response.status_code == 200, f'server recorded on error on reguest {full_url}'
    soup = BeautifulSoup(response.text)
    soups.append(soup)
    
lengths =[]
for soup in soups:
    lengths.append(len(soup.text))
print(lengths)
import matplotlib.pyplot as plt
plt.plot(lengths)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13157894736842105, 'fraction_numerical': 0.016333938294010888, 'mean_word_length': 3.803921568627451, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '137878', 'license': 'MIT', 'n_tokens_mistral': 770, 'n_tokens_neox': 725, 'n_words': 175, 'path': 'notebooks/scrape_wikipedia_1.0.ipynb', 'repo_name': 'ekand/presidential', 'size': '2215'}