<jupyter_start><jupyter_text>Amazon Scrapper  Requirements<jupyter_code>import csv
from bs4 import BeautifulSoup
import pandas as pd
from selenium import webdriver<jupyter_output><empty_output><jupyter_text>Starting Web Driver<jupyter_code>driver = webdriver.Chrome()<jupyter_output><empty_output><jupyter_text>Collection<jupyter_code>pages = range(1, 168)
df = pd.DataFrame()
failed_list = []
page_cnt = 1
for page in pages:
    link = "https://www.amazon.com/s?i=computers-intl-ship&bbn=16225007011&rh=n%3A16225007011%2Cn%3A13896617011%2Cn%3A565108&dc&page={}".format(page)
    driver.get(link)
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    results = soup.find_all('a', {"class": "a-link-normal a-text-normal"}, href=True)
    urls = []

    for html in results:
        href = html.get('href')
        href = 'https://www.amazon.com/' + href
        urls.append(href)
    cnt = 1
    mx = len(urls)
    for url in urls:
        attrs = {}
        attrs_failed = {}
        failed_str = ''
        driver.get(url)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        try:
            results = soup.find('span', {"id": "priceblock_ourprice"})
            attrs['price'] = results.text
        except:
            failed_str += 'price, '
        try:
            results = soup.find('span', {"class": "a-icon-alt"})
            attrs['rating'] = results.text[:3]
        except:
            failed_str += 'rating, '
        data = []
        data2 = []
        try:
            table = soup.find('table', attrs={'id':'productDetails_techSpec_section_1'})
            table_body = table.find('tbody')

            rows = table_body.find_all('tr')
            for row in rows:
                key = row.find_all('th')
                value = row.find_all('td')
                keys = [ele.text.strip() for ele in key]
                values = [ele.text.strip() for ele in value]
                data.append(keys[0])
                data2.append(values[0][1:])


            for i in range(len(data)):
                attrs[data[i]] = data2[i]
        except:
            failed_str += 'table_1, '
        data = []
        data2 = []
        try:
            table = soup.find('table', attrs={'id':'productDetails_techSpec_section_2'})
            table_body = table.find('tbody')

            rows = table_body.find_all('tr')
            for row in rows:
                key = row.find_all('th')
                value = row.find_all('td')
                keys = [ele.text.strip() for ele in key]
                values = [ele.text.strip() for ele in value]
                data.append(keys[0])
                data2.append(values[0][1:])

            for i in range(len(data)):
                attrs[data[i]] = data2[i]
        except:
            failed_str += 'table_2,'
        print('added item', cnt, ' out of ', mx, '(page {} out of 168)'.format(page_cnt))

        cnt += 1
        df = df.append(attrs, ignore_index=True)
        if failed_str != '':
             failed_list.append([failed_str, url])
    print(len(failed_list))
    page_cnt += 1<jupyter_output>added item 1  out of  24 (page 1 out of 168)
added item 2  out of  24 (page 1 out of 168)
added item 3  out of  24 (page 1 out of 168)
added item 4  out of  24 (page 1 out of 168)
added item 5  out of  24 (page 1 out of 168)
added item 6  out of  24 (page 1 out of 168)
added item 7  out of  24 (page 1 out of 168)
added item 8  out of  24 (page 1 out of 168)
added item 9  out of  24 (page 1 out of 168)
added item 10  out of  24 (page 1 out of 168)
added item 11  out of  24 (page 1 out of 168)
added item 12  out of  24 (page 1 out of 168)
added item 13  out of  24 (page 1 out of 168)
added item 14  out of  24 (page 1 out of 168)
added item 15  out of  24 (page 1 out of 168)
added item 16  out of  24 (page 1 out of 168)
added item 17  out of  24 (page 1 out of 168)
added item 18  out of  24 (page 1 out of 168)
added item 19  out of  24 (page 1 out of 168)
added item 20  out of  24 (page 1 out of 168)
added item 21  out of  24 (page 1 out of 168)
added item 22  out of  24 (page 1 out of 16[...]<jupyter_text>Create Dataframe<jupyter_code>df.describe()
df
df.to_csv('data.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11881656804733728, 'fraction_numerical': 0.056568047337278105, 'mean_word_length': 2.2308868501529053, 'pattern_counts': {'":': 3, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '665843', 'license': 'MIT', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1391, 'n_words': 436, 'path': 'Scrapper.ipynb', 'repo_name': 'mahdi-darvish/End-to-End-Price-Analysis-and-Prediction', 'size': '4225'}