<jupyter_start><jupyter_text><jupyter_code>!pip install kora -q
'''load packages'''
import requests
from bs4 import BeautifulSoup
from kora.selenium import wd
import pandas as pd
# Establish Session
url = 'https://www.amazon.com.au'
wd.get(url)
# Work out URL pattern
template = 'https://www.amazon.com.au/s?k={}&ref=nb_sb_noss_2'

search_term = 'ultrawide monitor'
search_term = search_term.replace(' ', '+')

url = template.format(search_term)
wd.get(url)
 def get_url(search_term):
  template = 'https://www.amazon.com.au/s?k={}&ref=nb_sb_noss_2'
  
  search_term = search_term.replace(' ', '+')
  url = template.format(search_term)

  return url
#search_term = input("What do you want to search for? ")

search_term = 'ultrawide monitor'
url = get_url(search_term)
wd.get(url)
soup = BeautifulSoup(wd.page_source, 'html.parser')<jupyter_output><empty_output><jupyter_text>Extract the Collection<jupyter_code>results = soup.find_all('div', {'data-component-type': 's-search-result'})
results[0].h2.a.text
def extract_description(item):
  description = item.h2.a.text
  return description
extract_description(results[12])<jupyter_output><empty_output><jupyter_text>Prototype the record<jupyter_code>item = results[0]
price_parent = item.find('span', 'a-price')
price = price_parent.find('span', 'a-offscreen')
price.text
def extract_price(item):
  try:
    price_parent = item.find('span', 'a-price')
    price = price_parent.find('span', 'a-offscreen')
    price = price.text
  except AttributeError:
    price = ''
  return price
extract_price(results[0])
def extract_record(item):
  return {
      'description': extract_description(item),
      'price': extract_price(item)
  }
extract_record(results[0])
for r in results:
  print(extract_record(r))
records = []
for r in results:
  records.append(extract_record(r))

records[0]

df = pd.DataFrame.from_records(records)
df.head()<jupyter_output><empty_output><jupyter_text>Generalize the pattern<jupyter_code>records = []
results = soup.find_all('div', {'data-component-type': 's-search-result'})

for item in results:
  records.append(extract_record(item))<jupyter_output><empty_output><jupyter_text>Error Handling<jupyter_code><jupyter_output><empty_output><jupyter_text>Find next page<jupyter_code><jupyter_output><empty_output><jupyter_text>Putting it all together<jupyter_code>!pip install kora -q
'''load packages'''
import requests
from bs4 import BeautifulSoup
from kora.selenium import wd
import pandas as pd

def get_url(search_term):
 template = 'https://www.amazon.com.au/s?k={}&ref=nb_sb_noss_2'
  
 search_term = search_term.replace(' ', '+')
 url = template.format(search_term)

 return url

def extract_description(item):
  description = item.h2.a.text
  return description

def extract_price(item):
  try:
    price_parent = item.find('span', 'a-price')
    price = price_parent.find('span', 'a-offscreen')
    price = price.text
  except AttributeError:
    price = ''
  return price

def extract_record(item):
  return {
      'description': extract_description(item),
      'price': extract_price(item)
  }
search_term = input('What do you want to search for? ')
url = get_url(search_term)
wd.get(url)
soup = BeautifulSoup(wd.page_source, 'html.parser')
records = []
results = soup.find_all('div', {'data-component-type': 's-search-result'})

for r in results:
  records.append(extract_record(r))

df = pd.DataFrame.from_records(records)
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14103677961193165, 'fraction_numerical': 0.004344048653344918, 'mean_word_length': 5.866799204771372, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '96111', 'license': 'MIT', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1216, 'n_words': 231, 'path': 'amazon_prices.ipynb', 'repo_name': 'Nicompag/ISYS5002_portfolio', 'size': '3453'}