<jupyter_start><jupyter_text>Notebook to parse the webpage for experimentations<jupyter_code>import os
import bs4

file_path = r"webpage.html"
with open(file_path, encoding="UTF-8") as f:
    soup = bs4.BeautifulSoup(f, features="lxml")
class_found = soup.find("div", class_="s-main-slot s-result-list s-search-results sg-row")
sponsored_products = class_found.find_all("div", class_="sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 AdHolder sg-col sg-col-4-of-20")
len(sponsored_products)
sponsored_products[0]
normal_products = class_found.find_all("div", class_="sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 sg-col sg-col-4-of-20")

len(normal_products)
normal_products[0]
normal_products[0].find_all(attrs={"name" : "data-component-props"})

normal_products[0].find_all("span", class_="rush-component")
ss = """<div class="sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 sg-col sg-col-4-of-20" data-asin="B075ZXXM84"
    data-cel-widget="search_result_1" data-component-id="14" data-component-type="s-search-result" data-index="1"
    data-uuid="d91507c9-ebbb-4f75-a310-21f9e89caeb1">
    <div class="sg-col-inner">
        <span cel_widget_id="MAIN-SEARCH_RESULTS-1"
            class="celwidget slot=MAIN template=SEARCH_RESULTS widgetId=search-results"
            data-cel-widget="MAIN-SEARCH_RESULTS-1" data-csa-c-id="i4n6w6-rs10pi-ey6b8j-373q6c">
            <div class="s-expand-height s-include-content-margin s-border-bottom s-latency-cf-section">
                <div class="a-section a-spacing-medium">
                    <div class="a-section a-spacing-micro s-grid-status-badge-container">
                        <span class="rush-component" data-component-id="15"
                            data-component-props='{"badgeType":"best-seller","asin":"B075ZXXM84"}'
                            data-component-type="s-status-badge-component">
                            <div class="a-row a-badge-region"><span
                                    aria-labelledby="B075ZXXM84-best-seller-label B075ZXXM84-best-seller-supplementary"
                                    class="a-badge" data-a-badge-supplementary-position="right"
                                    data-a-badge-type="status" id="B075ZXXM84-best-seller" tabindex="0"><span
                                        aria-hidden="true" class="a-badge-label" data-a-badge-color="sx-orange"
                                        id="B075ZXXM84-best-seller-label"><span
                                            class="a-badge-label-inner a-text-ellipsis">
                                            <span class="a-badge-text" data-a-badge-color="sx-cloud">Best Seller</span>
                                        </span></span><span aria-hidden="true"
                                        class="a-badge-supplementary-text a-text-ellipsis"
                                        id="B075ZXXM84-best-seller-supplementary">in Powder Baby Formula</span></span>
                            </div>
                        </span>
                    </div>
                    <span class="rush-component" data-component-id="16" data-component-type="s-product-image">
                        <a class="a-link-normal s-no-outline"
                            href="/Similac-Pro-Advance-Non-GMO-Formula-One-Month/dp/B075ZXXM84/ref=sr_1_2?dchild=1&amp;keywords=baby+milk&amp;qid=1608613874&amp;sr=8-2">
                            <div class="a-section aok-relative s-image-square-aspect">
                                <img alt="Similac Pro-Advance Non-GMO Infant Formula with Iron, with 2’-FL HMO, for Immune Support, Baby Formula, Powder, (One-Month..."
                                    class="s-image" data-image-index="1" data-image-latency="s-product-image"
                                    data-image-load="" data-image-source-density="1"
                                    src="https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL320_.jpg"
                                    srcset="https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL320_.jpg 1x, https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL480_FMwebp_QL65_.jpg 1.5x, https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL640_FMwebp_QL65_.jpg 2x, https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL800_FMwebp_QL65_.jpg 2.5x, https://m.media-amazon.com/images/I/71z5zJWD33L._AC_UL960_FMwebp_QL65_.jpg 3x" />
                            </div>
                        </a>
                    </span>
                    <div class="a-section a-spacing-none a-spacing-top-small">
                        <h2 class="a-size-mini a-spacing-none a-color-base s-line-clamp-4">
                            <a class="a-link-normal a-text-normal"
                                href="/Similac-Pro-Advance-Non-GMO-Formula-One-Month/dp/B075ZXXM84/ref=sr_1_2?dchild=1&amp;keywords=baby+milk&amp;qid=1608613874&amp;sr=8-2">
                                <span class="a-size-base-plus a-color-base a-text-normal" dir="auto">Similac Pro-Advance
                                    Non-GMO Infant Formula with Iron, with 2’-FL HMO, for Immune Support, Baby Formula,
                                    Powder, (One-Month Supply), 2.25 Pound (Pack of 3)</span>
                            </a>
                        </h2>
                        <div class="a-row a-size-base a-color-secondary">
                            <div class="a-row a-spacing-top-micro a-color-base"><span
                                    class="a-color-information a-text-bold" dir="auto">2.3 Pound (Pack of 3)</span>
                            </div>
                        </div>
                    </div>
                    <div class="a-section a-spacing-none a-spacing-top-micro">
                        <div class="a-row a-size-small">
                            <span aria-label="4.8 out of 5 stars">
                                <span class="a-declarative"
                                    data-a-popover='{"max-width":"700","closeButton":false,"position":"triggerBottom","url":"/review/widgets/average-customer-review/popover/ref=acr_search__popover?ie=UTF8&amp;asin=B075ZXXM84&amp;ref=acr_search__popover&amp;contextId=search"}'
                                    data-action="a-popover">
                                    <a class="a-popover-trigger a-declarative" href="javascript:void(0)"><i
                                            class="a-icon a-icon-star-small a-star-small-5 aok-align-bottom"><span
                                                class="a-icon-alt">4.8 out of 5 stars</span></i><i
                                            class="a-icon a-icon-popover"></i></a>
                                </span>
                            </span>
                            <span aria-label="3,911">
                                <a class="a-link-normal"
                                    href="/Similac-Pro-Advance-Non-GMO-Formula-One-Month/dp/B075ZXXM84/ref=sr_1_2?dchild=1&amp;keywords=baby+milk&amp;qid=1608613874&amp;sr=8-2#customerReviews">
                                    <span class="a-size-base" dir="auto">3,911</span>
                                </a>
                            </span>
                        </div>
                    </div>
                </div>
            </div>
        </span>
    </div>
</div>"""
soup1 = bs4.BeautifulSoup(ss, "lxml")
soup1.find("div", class_="sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 sg-col sg-col-4-of-20").attrs<jupyter_output><empty_output><jupyter_text>ASIN<jupyter_code>sponsored_products[0].attrs
normal_products[0].attrs<jupyter_output><empty_output><jupyter_text>Product name/title<jupyter_code>sponsored_products[0].find("span", class_="a-size-base-plus a-color-base a-text-normal").text
normal_products[10].find("span", class_="a-size-base-plus a-color-base a-text-normal").text<jupyter_output><empty_output><jupyter_text>Price<jupyter_code>sponsored_products[0].find_all("span", class_="a-price")
for p in sponsored_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("span", class_="a-price"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("span", class_="a-price"))))
for p in normal_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("span", class_="a-price"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("span", class_="a-price"))))<jupyter_output>[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
['$29.99$29.99']
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
['$52.74$52.74']
[]
[]
['$22.34$22.34', '$24.98$24.98']
['$19.98$19.98']
['$24.95$24.95']
[]
[]
['$39.99$39.99']
[]
['$22.33$22.33', '$24.98$24.98']
['$37.99$37.99', '$40.72$40.72']
[]
['$69.99$69.99']
[]
[]
[]
[]
[]
['$6.99$6.99']
[]
[]
[]
['$12.99$12.99']
['$133.98$133.98']<jupyter_text>Review Count<jupyter_code>normal_products[0].find("span", class_="a-size-base")
sponsored_products[1].find("span", class_="a-size-base")
for p in normal_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("span", class_="a-size-base"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("span", class_="a-size-base"))))
for p in sponsored_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("span", class_="a-size-base"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("span", class_="a-size-base"))))<jupyter_output>['483', '($0.26/Fl Oz)']
['519']
[]
['325', '($1.12/Ounce)']
['195', '($1.08/Ounce)']
[]
['905', '($1.14/Ounce)']
['248']
['1']
['208', '($5.99/Ounce)', '\n                Climate Pledge Friendly\n            ', 'See 1 certification', 'We’ve made it easier to discover products with sustainability certifications, as part of our commitment to help preserve the natural world. Time is fleeting.', 'PRODUCT CERTIFICATION (1)']
['3,688']
['65', '\n    Save $1.00\n']<jupyter_text>Rating<jupyter_code>sponsored_products[10].find_all("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom")[0].text
normal_products[10].find_all("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom")[0].text
for p in sponsored_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom"))))
for p in normal_products:
    # print(getattr(p.find("span", class_="a-price"), "text", None), p.find("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom"))

    print(list(map (lambda x: getattr(x, "text", None), p.find_all("i", class_="a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom"))))<jupyter_output>[]
[]
[]
[]
[]
[]
[]
['4.7 out of 5 stars']
[]
['4.7 out of 5 stars']
['4.7 out of 5 stars']
['4.6 out of 5 stars']
[]
['4.6 out of 5 stars']
[]
[]
['4.5 out of 5 stars']
['4.6 out of 5 stars']
[]
[]
[]
[]
['4.6 out of 5 stars']
[]
[]
['4.6 out of 5 stars']
[]
['4.6 out of 5 stars']
['4.4 out of 5 stars']
[]
[]
['4.5 out of 5 stars']
['4.7 out of 5 stars']
['4.7 out of 5 stars']
['4.7 out of 5 stars']
['4.6 out of 5 stars']
['4.6 out of 5 stars']
[]
['4.6 out of 5 stars']
[]
['4.7 out of 5 stars']
[]
['4.7 out of 5 stars']
['4.7 out of 5 stars']
['4.4 out of 5 stars']
[]
['4.3 out of 5 stars']
[]<jupyter_text>Pagination<jupyter_code>page_links = [url_link for url_link in (link.find("li", class_="a-normal") for link in soup.find_all("ul", class_="a-pagination")) if url_link]
page_links
 [link.find("li", class_="a-normal") for link in soup.find_all("ul", class_="a-pagination")]
soup.find("ul", class_="a-pagination")
 for link in soup.find("ul", class_="a-pagination").find_all("li", class_=["a-normal", "a-last"]):
     print(link.find("a").attrs)
     print()
len(soup.find_all("ul", class_="a-pagination"))
 for link in soup.find("ul", class_="a-pagination").find_all("li", class_=["a-last"]):
     print(link.find("a").attrs)
     print()


def parse_data(file_path):
    with open(file_path, "r", encoding="utf-8") as f:
        file_data = f.read()
    
    soup = bs4.BeautifulSoup(file_data, "lxml")

    products = []

    main_result_class = "s-main-slot s-result-list s-search-results sg-row"
    sponsored_products_class = "sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 AdHolder sg-col sg-col-4-of-20"
    normal_products_class = "sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 sg-col sg-col-4-of-20"

    product_name_class = "a-size-base-plus a-color-base a-text-normal"
    price_class = "a-price"
    review_count_class = "a-size-base"
    rating_class = "a-icon a-icon-star-small a-star-small-4-5 aok-align-bottom"

    products_found = soup.find("div", class_=main_result_class)
    if products_found:
        sponsored_products = class_found.find_all("div", class_=sponsored_products_class)
        normal_products = class_found.find_all("div", class_=normal_products_class)

        for sponsored_product in sponsored_products:
            products.append(
                {
                    "asin": sponsored_product.attrs.get("data-asin", None),
                    "product_name": getattr(sponsored_product.find("span", class_=product_name_class), "text", None),
                    "price": getattr(sponsored_product.find("span", class_=price_class), "text", None),
                    "sponsored": True,
                    "review_count": getattr(sponsored_product.find("span", class_=review_count_class), "text", None),
                    "average_ratings": getattr(sponsored_product.find("i", class_=rating_class), "text", None),

                }
            )

        for sponsored_product in normal_products:
            products.append(
                {
                    "asin": sponsored_product.attrs.get("data-asin", None),
                    "product_name": getattr(sponsored_product.find("span", class_=product_name_class), "text", None),
                    "price": getattr(sponsored_product.find("span", class_=price_class), "text", None),
                    "sponsored": False,
                    "review_count": getattr(sponsored_product.find("span", class_=review_count_class), "text", None),
                    "average_ratings": getattr(sponsored_product.find("i", class_=rating_class), "text", None),

                }
            )

            return products
parse_data(file_path="webpage.html")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1735241502683363, 'fraction_numerical': 0.03908077611118756, 'mean_word_length': 2.3459944751381214, 'pattern_counts': {'":': 18, '<': 95, '<?xml version=': 0, '>': 95, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '554771', 'license': 'Apache-2.0', 'n_tokens_mistral': 5393, 'n_tokens_neox': 4976, 'n_words': 782, 'path': 'parse_page.ipynb', 'repo_name': 'kartikeyporwal/amazon-product-scrapper', 'size': '14534'}