<jupyter_start><jupyter_text>Option 1: Using Beautiful soup "the html way"<jupyter_code>soup.title
soup.title.name
soup.title.string
soup.title.parent
soup.title.parent.name
soup.title.parent.string
soup.p
soup.p['class']
soup.find_all('p')
soup.find_all('a')[0].get('href')
for atag in soup.find_all('a'):
    print(atag.get('href'))<jupyter_output>http://example.com/elsie
http://example.com/lacie
http://example.com/tillie<jupyter_text>Option 2: Using Beautiful soup "the CSS way"<jupyter_code>soup.select('#link1')
soup.select('a')[0].text
soup.select('a')[0].get_text()
for atag in soup.select('a'):
    print(atag.text)
soup.select('a')[0].text<jupyter_output><empty_output><jupyter_text>A css selector address:```html.lister-list > tr:nth-child(7) > td:nth-child(2) > a:nth-child(1)```  Let's scrape the imdb Top 250<jupyter_code># 1. importing libraries
from bs4 import BeautifulSoup
import requests
import pandas as pd

# 2. find url and store it in avariable
url = "https://www.imdb.com/chart/top"

# 3. download html with a get request
response = requests.get(url)
response.status_code
soup = BeautifulSoup(response.content, 'html.parser')
soup.select(".lister-list > tr:nth-child(1) > td:nth-child(2) > a:nth-child(1)")
soup.select("td.titleColumn a")
# Movie title
soup.select("td.titleColumn a")[0].text
# Actors / actresses and dir' stored in 'title' attribute
soup.select("td.titleColumn a")[0]['title']
soup.select("td.titleColumn span.secondaryInfo")[0].text<jupyter_output><empty_output><jupyter_text>Now make it work for all the movies<jupyter_code># movie_lst = soup.select("td.titleColumn a")
# yr_lst = soup.select("td.titleColumn span.secondaryInfo")
from tqdm.notebook import tqdm

title = []
dir_stars = []
year = []

len_movies = len(movie_lst)

for i in tqdm(range(len_movies)):
    title.append(soup.select("td.titleColumn a")[i].text)
    dir_stars.append(soup.select("td.titleColumn a")[i]['title'])
    year.append(soup.select("td.titleColumn span.secondaryInfo")[i].text)
title
dir_stars
year
# clean the year column
year_clean = [yr.strip(')').strip('(') for yr in year]
year_clean
# we're going to split the dir_star list into two actor lists
# and one director list
actor1 = []
actor2 = []
dir1 = []

for dir_star in dir_stars:
    temporary_people = dir_star.split(",")  # only three
    
    # now append the individual people to their corresponding lists
    # (already cleaned!)
    dir1.append(temporary_people[0].replace(' (dir.)', ''))
    actor1.append(temporary_people[1][1:])
    actor2.append(temporary_people[2][1:])
# create the resulting dataframe
movies = pd.DataFrame({'year': year_clean,
                       'title': title,
                       'dir': dir1,
                       'actor1': actor1,
                       'actor2': actor2,
                      })
movies<jupyter_output><empty_output><jupyter_text>Sraping many pages<jupyter_code>url = 'https://www.imdb.com/search/title/?title_type=feature&release_date=1990-01-01,1992-12-31&user_rating=7.5'
from bs4 import BeautifulSoup
import requests
import pandas as pd
response = requests.get(url)
response.status_code
response.content
# html 
soup = BeautifulSoup(response.content, 'html.parser')
soup
# define a start variable that goes into the url
iterations = range(1,501, 50)
#s_var = 101
f_url = [f"https://www.imdb.com/search/title/?title_type=feature&release_date=1990-01-01,1992-12-31&user_rating=7.5,&start={i}&ref_=adv_nxt" for i in iterations]
for i in iterations:
    print(i)
f_url = [f"https://www.imdb.com/search/title/?title_type=feature&release_date=1990-01-01,1992-12-31&user_rating=7.5,&start={i}&ref_=adv_nxt" for i in iterations]<jupyter_output><empty_output><jupyter_text>Respectful scraping!<jupyter_code>from time import sleep 
for i in range (5):
    print(i)
    wait_time = randint(1,4)
    print("I will sleep for" + str(wait_time)+ "seconds")
    sleep(wait_time)
pages.append(reponse)
wait_time
from time import sleep 
for i in range (5):
    print(i)
    wait_time = randint(1,4)
    print("I will sleep for" + str(wait_time)+ "seconds")
    sleep(wait_time)
pages = []
for i in iterations:
    start_at = str(i)
    url = f"https://www.imdb.com/search/title/?title_type=feature&release_date=1990-01-01,1992-12-31&user_rating=7.5,&start={start_at}&ref_=adv_nxt"
#download the url
    response = requests.get(url)
# print the status code
    print("Status code: " + str(response.status_code))
    pages.append(response)
    wait_time = randint(0.300, 2)
    sleep(wait_time)<jupyter_output>Status code: 200<jupyter_text>Put everything together<jupyter_code>pages = []

for i in iterations:
    start_at = str(i)
    url = f_url = f"https://www.imdb.com/search/title/?title_type=feature&release_date=1990-01-01,1992-12-31&user_rating=7.5,&start={i}&ref_=adv_nxt"
     #download the url
    response = requests.gets<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14785276073619633, 'fraction_numerical': 0.03128834355828221, 'mean_word_length': 4.754117647058823, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '216870', 'license': 'Apache-2.0', 'n_tokens_mistral': 1884, 'n_tokens_neox': 1777, 'n_words': 349, 'path': 'code/8_01_intro_to_webscraping.ipynb', 'repo_name': 'petergeorge649/Websraping-multiple-pages', 'size': '4890'}