<jupyter_start><jupyter_text>Web scraping support materials  import of libraries<jupyter_code>import pandas as pd
import requests
from bs4 import BeautifulSoup
import re
import json<jupyter_output><empty_output><jupyter_text>main url<jupyter_code>base_url = "https://www.imdb.com"
movie_url = "https://www.imdb.com/title/tt2467372/"<jupyter_output><empty_output><jupyter_text>request and pass to beautiful soup<jupyter_code>r = requests.get(url=movie_url)

r
# create a BeautifulSoup object
soup = BeautifulSoup(r.text, 'html.parser')
soup.title
soup.find_all('a')
data = {}
title = soup.find('title')
print(title.string)
data["title"] = title.string

ratingValue = soup.find("span", {"itemprop" : "ratingValue"})
data["ratingValue"] = ratingValue.string

ratingCount = soup.find("span", {"itemprop" : "ratingCount"})
data["ratingCount"] = ratingCount.string

data<jupyter_output><empty_output><jupyter_text>... and many more features  Example with find<jupyter_code>data = json.loads(soup.find('script', type='application/ld+json').text)
print (data['actor'][0])
new_url = base_url + data['actor'][0]['url']
new_url
r = requests.get(url=new_url)

# create a BeautifulSoup object
soup = BeautifulSoup(r.text, 'html.parser')
soup
data2 = (soup.find("div", {"id": "filmography"}).text)
print (data2)<jupyter_output>Hide 
Show 
Actor (95 credits)




 2021

Hotel Transylvania 4
(announced)

Jonathan (voice)



 2022

Chip 'n' Dale: Rescue Rangers
(pre-production)

Dale (voice)



 2013-2021

Brooklyn Nine-Nine
(TV Series)

<NAME>

- Episode #8.5
(2021)
... <NAME>


- Episode #8.4
(2021)
... <NAME>ta


- Episode #8.3
(2021)
... <NAME>


- Episode #8.2
(2021)
... <NAME>


- Episode #8.1
(2021)
... <NAME>





Show all 153 episodes




 2020

Late Night with <NAME>
(TV Series)

The Mummy

- <NAME>/<NAME>/<NAME>/<NAME>
(2020)
... The Mummy




 2020

<NAME>
(TV Series short)

Bran BcBillan / Baby

- Killer Voices - Part 2
(2020)
... Bran BcBillan / Baby


- Killer Voices - Part 1
(2020)
... Bran BcBillan




 2020

Never Have I Ever
(TV Series)

<NAME>

- ...been the loneliest boy in the world
(2020)
... <NAME> (voice)




 2020

Palm Springs

Nyles



 2019

The Dark Crystal: Age of Resistance
(TV Seri[...]<jupyter_text>Agent rotation<jupyter_code>import requests
import random
user_agent_list = [
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
]
url = 'https://httpbin.org/headers'
for i in range(1,4):
    # Pick a random user agent
    user_agent = random.choice(user_agent_list)
    # Set the headers 
    headers = {'User-Agent': user_agent}
    # Make the request
    response = requests.get(url,headers=headers)
    
    print("Request #%d\nUser-Agent Sent:%s\n\nHeaders Recevied by HTTPBin:"%(i,user_agent))
    print(response.json())
    print("-------------------")<jupyter_output>Request #1
User-Agent Sent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36

Headers Recevied by HTTPBin:
{'headers': {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Host': 'httpbin.org', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36', 'X-Amzn-Trace-Id': 'Root=1-6002abe0-18fa9da35f21e0b02d12c815'}}
-------------------
Request #2
User-Agent Sent:Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0

Headers Recevied by HTTPBin:
{'headers': {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Host': 'httpbin.org', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0', 'X-Amzn-Trace-Id': 'Root=1-6002abe1-4edc12cc0925453041dbee01'}}
-------------------
Request #3
User-Agent Sent:Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0

Headers Rec[...]<jupyter_text>IP and proxies<jupyter_code>import requests
from itertools import cycle
import traceback
proxies = {'172.16.58.3:3128', '172.16.31.10:45169', '192.168.127.12:8080', '172.16.58.3:3128', '192.168.3.11:8080', '172.16.58.3:3128', '172.16.58.3:8080'}
proxy_pool = cycle(proxies)

url = 'https://httpbin.org/ip'
for i in range(1,11):
    # Get a proxy from the pool
    proxy = next(proxy_pool)
    print("Request #%d"%i)
    try:
        response = requests.get(url,proxies={"http": proxy, "https": proxy})
        print(response.json())
    except:
        # Most free proxies will often get connection errors. You will have retry the entire request using another proxy to work.  
        print("Skipping. Connnection error")<jupyter_output>Request #1
Skipping. Connnection error
Request #2
Skipping. Connnection error
Request #3
Skipping. Connnection error
Request #4
Skipping. Connnection error
Request #5
Skipping. Connnection error
Request #6
Skipping. Connnection error
Request #7
Skipping. Connnection error
Request #8
Skipping. Connnection error
Request #9
Skipping. Connnection error
Request #10
Skipping. Connnection error<jupyter_text>Selenium<jupyter_code>from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
# base url I want to start at
base_url = 'https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/103975/science-1'
# initiate a webdriver
DRIVER = 'chromedriver'
driver = webdriver.Chrome(DRIVER)
# get into the site
driver.get(base_url)
url = base_url
print(driver.current_url)
while True:
    driver.get(url)
    # get the name of the chapter
    chapter_name = url.split('/')[8]
    # save the HTML content locally for future processing
    with open("html_chapters/" + chapter_name + ".html" , "w", encoding="utf-8") as f:
        f.write(driver.page_source)
    # click on the arrow next to get on the next page
    elements = driver.find_elements_by_class_name("fa-chevron-double-right")
    for e in elements:
        # perform click
        e.click() 
    print(driver.current_url)
    if url == driver.current_url:
        break
    url = driver.current_url<jupyter_output>https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/103975/science-1
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/104266/science-2
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/104534/science-3
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/104880/science-4
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/105223/science-5
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/105610/science-6
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/105742/science-7
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/105839/science-8
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/107161/interlude-justice-demands-retribution
https://www.royalroad.com/fiction/8894/everybody-loves-large-chests/chapter/107872/gainful-employment-1
https://www.royalroad.com/ficti[...]
{'fraction_non_alphanumeric': 0.1503267973856209, 'fraction_numerical': 0.08104575163398693, 'mean_word_length': 4.159136884693189, 'pattern_counts': {'":': 3, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 16, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 7, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '402396', 'license': 'MIT', 'n_tokens_mistral': 3270, 'n_tokens_neox': 2842, 'n_words': 615, 'path': 'webscraping/project_web_scraping.ipynb', 'repo_name': 'lubojr/pyladies_vienna', 'size': '7757'}