<jupyter_start><jupyter_text>Web scrapingThis notebook demonstrates web scraping a Golf website using theBeautiful Soup package.Note: some websites will block scraping.  Web pageWe will use the [Golf Nationwide](http://golfnationwide.com/) website as an example.Let's say we want to gather the information of all the following into a CSV:* Course Name* Street* City* County* Course websiteThe home page for this website looks like:TODO: add photo.<jupyter_code># Load libaries
import requests
import pandas as pd
import re
import urllib.parse
from bs4 import BeautifulSoup
# Download the website using requests.get
url = 'http://golfnationwide.com/'
page = requests.get(url)
page
# Print out the page content
page.content

# Pass the page content to beautiful soup
soup = BeautifulSoup(page.content, 'html.parser')
print(soup.prettify())
# TODO: highlight the part of the HTML that shows "HomeCourseDirectory"
home_course_soup = soup.find(id='HomeCourseDirectory')
home_course_soup
# Descend into these by looking for links
state_soup = home_course_soup.find_all('a')
state_links = [link.get('href') for link in state_soup]
# These are relative paths, so we join the links
num_state_links = len(state_links)
state_links_abs_path = [urllib.parse.urljoin(url, state_link) for
                        state_link in state_links]
state_links_abs_path[:5]

# Define function to parse single state page
# You end up requiring quite a lot of hard-coding
# There might be better ways to smartly guess the div / tag numbers based on examples
single_page_url = 'http://golfnationwide.com/Golf-Courses-By-State/Alabama/Tartan-Pines-Golf-Club___17121.aspx'
single_page_url = 'http://golfnationwide.com/Golf-Courses-By-State/Alabama/Albertville-Golf-and-Country-Club___17020.aspx'
course_page = requests.get(single_page_url)
course_soup = BeautifulSoup(course_page.content, 'html.parser')
course_info_soup = course_soup.find('div', id='BasicInfo')
print(course_info_soup.prettify())
# Grab hard-coded values from attributes
course_name_soup = course_info_soup.find('span',
    id='ctl00_MainContentPlaceholder_CourseNameLabel')
course_name = course_name_soup.find(text=True, recursive=False)
course_name
address_name_soup = course_info_soup.find('span',
    id='ctl00_MainContentPlaceholder_AddressLabel')
address_name = address_name_soup.find(text=True, recursive=False)
address_name
website_soup = course_info_soup.find('a',
    id='ctl00_MainContentPlaceholder_WebAddressLink')
# Could use find_all in the case that it doesn't exist.
website_url = website_soup.get('href')
website_url<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08797541298501729, 'fraction_numerical': 0.006915097963887821, 'mean_word_length': 6.153846153846154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '284314', 'license': 'MIT', 'n_tokens_mistral': 849, 'n_tokens_neox': 809, 'n_words': 228, 'path': 'data_extraction/scraping.ipynb', 'repo_name': 'charlesincharge/python-examples', 'size': '2603'}