<jupyter_start><jupyter_text>Load Libraries<jupyter_code>import requests
from bs4 import BeautifulSoup as bs
import re
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load Page Content<jupyter_code># Load the content
r = requests.get('https://keithgalli.github.io/web-scraping/example.html')

# Convert into a BS object
soup = bs(r.content)
print(soup.prettify())<jupyter_output><html>
 <head>
  <title>
   HTML Example
  </title>
 </head>
 <body>
  <div align="middle">
   <h1>
    HTML Webpage
   </h1>
   <p>
    Link to more interesting example:
    <a href="https://keithgalli.github.io/web-scraping/webpage.html">
     keithgalli.github.io/web-scraping/webpage.html
    </a>
   </p>
  </div>
  <h2>
   A Header
  </h2>
  <p>
   <i>
    Some italicized text
   </i>
  </p>
  <h2>
   Another header
  </h2>
  <p id="paragraph-id">
   <b>
    Some bold text
   </b>
  </p>
 </body>
</html><jupyter_text>Find & FindAll<jupyter_code># Find individual tags

first_header = soup.find('h2')
print(first_header)

headers = soup.find_all('h2')
print(headers)
# Find a list of tags

first_headers = soup.find(['h1', 'h2'])
print(first_headers)

headers = soup.find_all(['h1', 'h2'])
print(headers)
# Pass attributes to BS

paragraph = soup.find_all('p', attrs = {'id': 'paragraph-id'})
print(paragraph)
# Nest through find/find_all calls
body = soup.find('body')
div = body.find('div')
header = div.find('h1')
print(header)
# Search for spesific strings

paragraphs = soup.find_all('p', string = re.compile('Some'))
print(paragraphs)

headers = soup.find_all('h2', string = re.compile('(h|H)eader'))
print(headers)<jupyter_output>[<p><i>Some italicized text</i></p>, <p id="paragraph-id"><b>Some bold text</b></p>]
[<h2>A Header</h2>, <h2>Another header</h2>]<jupyter_text>CSS Selector<jupyter_code>soup.body
content = soup.select('div p')
print(content)
paragraphs = soup.select('h2 ~ p')
print(paragraphs)
paragraphs = soup.select('p#paragraph-id b')
print(paragraphs)
paragraphs = soup.select('body > p ')
print(paragraphs)
# Get by css property
alignment = soup.select('[align=middle]')
print(alignment)<jupyter_output>[<div align="middle">
<h1>HTML Webpage</h1>
<p>Link to more interesting example: <a href="https://keithgalli.github.io/web-scraping/webpage.html">keithgalli.github.io/web-scraping/webpage.html</a></p>
</div>]<jupyter_text>Get HTML properties<jupyter_code>header = soup.find('h2')
print(header.string)

# Gets a little tricky with nested elements
div = soup.find('div')
print(div.get_text())
link = soup.find('a')
print(link['href'])

paragraph = soup.select('p#paragraph-id')
print(paragraph[0]['id'])<jupyter_output>https://keithgalli.github.io/web-scraping/webpage.html
paragraph-id<jupyter_text>Code Navigation<jupyter_code>soup.body.div.p.a
# Terms: Parent, Siblings, Childs

div = soup.body.find('div').find_next_siblings()
print(div)<jupyter_output>[<h2>A Header</h2>, <p><i>Some italicized text</i></p>, <h2>Another header</h2>, <p id="paragraph-id"><b>Some bold text</b></p>]<jupyter_text>Practices  Practice 1Grab all social links on the webpage<jupyter_code># Load page
r = requests.get('https://keithgalli.github.io/web-scraping/webpage.html')

soup = bs(r.content)
# Using select

socials = [tag['href'] for tag in soup.select('ul.socials li a')]
print(socials)
# Using find
socials = [tag['href'] for tag in soup.find('ul', attrs = {'class': 'socials'}).find_all('a')]
print(socials)<jupyter_output>['https://www.instagram.com/keithgalli/', 'https://twitter.com/keithgalli', 'https://www.linkedin.com/in/keithgalli/', 'https://www.tiktok.com/@keithgalli']<jupyter_text>Practice 2Scrape the MIT table<jupyter_code>table = soup.select('table.hockey-stats')[0]
columns = table.find('thead').find_all('th')

column_names = [column.string for column in columns]

rows = table.find('tbody').find_all('tr')
data = []
for row in rows:
    td = row.find_all('td')
    data.append([str(data.get_text()).strip() for data in td])

dataset = pd.DataFrame(data, columns = column_names)
dataset<jupyter_output><empty_output><jupyter_text>Practice 3Grab all fun facts that contain 'is'<jupyter_code>facts = soup.select('ul.fun-facts li')
facts = [fact.find_all(string = re.compile('is')) for fact in facts]
facts = [fact for fact in facts if fact != []]
print(facts)<jupyter_output>[['Middle name is Ronald'], ['<NAME>uts coffee is better than Starbucks'], ['A favorite book series of mine is '], ['Current video game of choice is '], ["The band that I've seen the most times live is the "]]<jupyter_text>Practice 3Download an Image<jupyter_code>img = soup.select('.row .column img')[0]
img_url = 'https://keithgalli.github.io/web-scraping' + '/' + img['src']

img_data = requests.get(img_url).content
with open('lake_como.jpg', 'wb') as handler:
    handler.write(img_data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17314853123707075, 'fraction_numerical': 0.00703351261894911, 'mean_word_length': 5.524966261808367, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 96, 'https://': 10, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '428011', 'license': 'Apache-2.0', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1805, 'n_words': 370, 'path': 'Courses/Keith Galli/Comprehensive Python Beautiful Soup Web Scraping Tutorial/Comprehensive Python Beautiful Soup Web Scraping Tutorial.ipynb', 'repo_name': 'keivanipchihagh/Intro_to_DS_and_ML', 'size': '4838'}