<jupyter_start><jupyter_text>Part 1 of creating auto-populate feature Get the URLs for the recipes on allrecipes.com Import BeautifulSoup and requests<jupyter_code>from bs4 import BeautifulSoup
import requests
# I'm trying to get all of the recipes off of allrecipes.com.
# This is the sitemap of the recipe categories.
main_url = 'https://www.allrecipes.com/recipes/'
# Make a call to get the html of main_url
response = requests.get(main_url)
soup = BeautifulSoup(response.text, "lxml")
# Uncomment soup and run if you want to see what is returned.
#soup
# Just checking the title of the webpage.
soup.title.text<jupyter_output><empty_output><jupyter_text>Get the links for each of the main categories<jupyter_code># They are contained in the class "col-container"
soup.find('div', class_='col-container')
# The recipes are in three columns divided into 'all-categories-col'. 
# This is just another way to get all of the parent links.
categories = soup.find_all('div', class_='all-categories-col')
categories
# Parent links for different categories of recipes
parent_cat_urls = []

for ul in soup.find_all('div', class_='all-categories-col'):
  for li in ul.find_all('li'):
    a = li.find('a')
    print(a['href'], a.get_text())
    parent_cat_urls.append(a['href'])
parent_cat_urls
len(parent_cat_urls)
# Drop the brand pages link because it doesn't contain recipes
parent_cat_urls = parent_cat_urls[:-1]
len(parent_cat_urls)<jupyter_output><empty_output><jupyter_text>Move into the categories starting with breakfast as a test.<jupyter_code># Getting the html from the parent breakfast category.
response = requests.get(parent_cat_urls[0])
soup = BeautifulSoup(response.text, "lxml")
# The div class 'grid slider' has all of the sub-categories for breakfast recipes
categories = soup.find_all('div', class_='grid slider')
categories
# Calling these sub-categories child-categories of the parents.
child_cat_urls = []

for ul in soup.find_all('div', class_='grid slider'):
  for li in ul.find_all('li'):
    a = li.find('a')
    print(a['href'], a.get_text())
    child_cat_urls.append(a['href'])
child_cat_urls<jupyter_output><empty_output><jupyter_text>Get grandchild urls from pancakes<jupyter_code># Get html of the pancakes page.
response = requests.get(child_cat_urls[0])
soup = BeautifulSoup(response.text, "lxml")
# These pages are set up just like the child categories with a grid slider class
# along the top of the page with yet more sub-categories.
categories = soup.find_all('div', class_='grid slider')
categories
# Grandchildren categories from the parents.
grandchild_cat_urls = []

for ul in soup.find_all('div', class_='grid slider'):
  for li in ul.find_all('li'):
    a = li.find('a')
    print(a['href'], a.get_text())
    grandchild_cat_urls.append(a['href'])
grandchild_cat_urls<jupyter_output><empty_output><jupyter_text>Get URLs for recipes in buttermilk pancakes<jupyter_code># From here we get to actual recipes on allrecipes.com
# Get the html for grandchildren category.
response = requests.get(grandchild_cat_urls[0])
soup = BeautifulSoup(response.text, "lxml")
# The recipe links are contained in the class 'fixed-recipe-card__info'
recipes = soup.find_all('div', class_='fixed-recipe-card__info')
recipes
# Exploring how to get to the link for a recipe of Spelt Pancakes.
card = soup.find('div', class_='fixed-recipe-card__info')
link = card.h3.a['href']
link
# Combining above exploration into one line.
soup.find('div', class_='fixed-recipe-card__info').h3.a['href']
# List to store links to recipes.
recipe_urls = []

for article in soup.find_all('div', class_='fixed-recipe-card__info'):
  link = article.h3.a['href']
  print(link)
  recipe_urls.append(link)
recipe_urls<jupyter_output><empty_output><jupyter_text>The pages are infinite scrolling so I'm missing recipes and want more than the 'first' page shows automatically.<jupyter_code># Make a set to store recipe urls. A set will make it so a given recipe is only
# saved once in case it's stored in more than one category.
recipe_urls_set = set()
# Set up a loop to check for valid webpages to move through infinite scrolling
i = 1  # Start with page 1

while True:
  # I got this url in the Request Headers section of the network dev tools
  # in Google Chrome after selecting just XHR.
  url = f'https://www.allrecipes.com/recipes/22951/breakfast-and-brunch/pancakes/buttermilk-pancakes/?page={i}'
  r = requests.get(url)
  #print(r)
  if r.status_code == 404:  # Check if 404 error when trying to load the ith page.
    break  # Break out of loop if ith page doesn't exist.
  else:  # Continue if it does.
    soup = BeautifulSoup(r.text, "lxml")
    #print(i)

    i += 1  # Increment to go to next page.


    for article in soup.find_all('div', class_='fixed-recipe-card__info'):
      link = article.h3.a['href']
      recipe_urls_set.add(link)  # Add the link to set if it's new.

print(len(recipe_urls_set))
recipe_urls_set<jupyter_output>69<jupyter_text>Looks like I ended up with a set of 69 pancake recipes. That's pretty sweet.  Time to try getting all of the recipes using the above logic and loops.<jupyter_code># I want to be kind to the allrecipes.com servers so include pauses in scraping.
import time
t = 0.5
# All of the parent categories on https://www.allrecipes.com/recipes/
len(parent_cat_urls)
# Set of child categories.
child_cat_urls_set = set()
for i in range(len(parent_cat_urls)):
  response = requests.get(parent_cat_urls[i])
  soup = BeautifulSoup(response.text, "lxml")
  print(f"Getting children links for {parent_cat_urls[i]}. {i+1} of 92.")

  for ul in soup.find_all('div', class_='grid slider'):
    for li in ul.find_all('li'):
      a = li.find('a')
  #    print(a['href'], a.get_text())
      child_cat_urls_set.add(a['href'])
  
  time.sleep(t) # Pause before continuing
len(child_cat_urls_set)
# Grandchild categories set.
grandchild_cat_urls_set = set()
for i, child_cat in enumerate(child_cat_urls_set):
  response = requests.get(child_cat)
  soup = BeautifulSoup(response.text, "lxml")
  print(f"Getting grand-children links for {child_cat}. {i+1} of 974.")

  for ul in soup.find_all('div', class_='grid slider'):
    for li in ul.find_all('li'):
      a = li.find('a')
      #print(a['href'], a.get_text())
      grandchild_cat_urls_set.add(a['href'])

  time.sleep(t) # Pause before continuing
# Total grandchild categories where the recipes are actually kept.
len(grandchild_cat_urls_set)<jupyter_output><empty_output><jupyter_text>The category urls are stored. Now get the recipe urls.<jupyter_code># Set for all of the recipe urls and to avoid duplicates
recipe_set = set()<jupyter_output><empty_output><jupyter_text>This next cell is going to take a LONG time.<jupyter_code>for j, grandchild_url in enumerate(grandchild_cat_urls_set):
  print(f'Getting recipe links from {grandchild_url}. {j+1} of 1708')
  i = 1

  while True:
    url = f'{grandchild_url}?page={i}'
    r = requests.get(url)
    #print(r)
    if r.status_code == 404:
      break
    else:
      soup = BeautifulSoup(r.text, "lxml")
      #print(i)

      i += 1

      for article in soup.find_all('div', class_='fixed-recipe-card__info'):
        link = article.h3.a['href']
        recipe_set.add(link)

      time.sleep(0.5)
# How many recipes total?
len(recipe_set)
recipe_set
import csv
# Save the recipe urls to a csv for later use.
with open('allrecipes_urls.csv', 'w', newline='') as f:
  thewriter = csv.writer(f)

  for recipe in recipe_set:
    thewriter.writerow([recipe])
# csv_file = open('allrecipes.csv', 'w')

# csv_writer = csv.writer(csv_file)

# for url in recipe_set:
#   csv_writer.writerow([url])

# csv_file.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11340876576928079, 'fraction_numerical': 0.006502796202367018, 'mean_word_length': 4.721726190476191, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '209528', 'license': 'MIT', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2594, 'n_words': 790, 'path': 'Dishify/notebooks/ingredient_populater/1_allrecipes_url_web_scraper.ipynb', 'repo_name': 'NealWhitlock/MyDish-DS', 'size': '7689'}