<jupyter_start><jupyter_text>Importing the required libraries<jupyter_code>import pandas as pd
import requests
from bs4 import BeautifulSoup
import time<jupyter_output><empty_output><jupyter_text>Getting the raw html using the get_soup() function<jupyter_code>def get_soup(url):
    proxylist = ['192.168.3.11:1080', '172.16.17.32:47638', '172.16.31.10:1080', '192.168.127.12:59427', '172.16.17.32:9988', '192.168.127.12:48540', '172.16.58.3:37884', '172.16.17.32:1080', '172.16.31.10:1081', '192.168.127.12:65432', '172.16.58.3:49228', '192.168.3.11:1080', '172.16.17.32:3550', '172.16.17.32:6111', '192.168.3.11:9300', '192.168.3.11:6111', '192.168.127.12:1080', '192.168.127.12:1080', '172.16.17.32:31870', '172.16.58.3:54892', '192.168.3.11:1080', '192.168.127.12:9050', '172.16.17.32:3598', '172.16.58.3:10808', '192.168.127.12:8080', '172.16.31.10:9100', '172.16.17.32:37884', '172.16.31.10:64935', '172.16.17.32:31337', '172.16.31.10:24260', '192.168.127.12:9050', '192.168.127.12:30588', '172.16.31.10:21945', '192.168.127.12:4145', '192.168.127.12:1088', '192.168.127.12:1080', '172.16.17.32:4145', '172.16.58.3:1088', '172.16.17.32:62625', '192.168.3.11:999', '192.168.3.11:9050', '172.16.17.32:10086', '172.16.31.10:1080', '192.168.3.11:1080', '172.16.31.10:58890', '192.168.127.12:9507', '172.16.31.10:6111', '172.16.17.32:3080', '192.168.3.11:1080', '172.16.31.10:48540', '172.16.17.32:8085', '172.16.17.32:57848', '172.16.58.3:6111', '172.16.17.32:9608', '172.16.58.3:1080', '192.168.127.12:6111', '172.16.31.10:1080', '172.16.31.10:25503', '172.16.31.10:38067', '192.168.127.12:55056', '172.16.31.10:1080', '192.168.127.12:57745', '192.168.127.12:37884', '172.16.58.3:8888', '192.168.3.11:1080', '172.16.58.3:24260', '192.168.3.11:6111', '172.16.58.3:1090', '172.16.17.32:37884', '172.16.31.10:6111', '172.16.58.3:1080', '172.16.58.3:9050', '172.16.58.3:1080', '192.168.127.12:4145', '172.16.31.10:33388', '172.16.31.10:1080', '192.168.127.12:6111', '192.168.127.12:1157', '172.16.17.32:3565', '172.16.31.10:1080', '172.16.58.3:6111', '172.16.31.10:4145', '172.16.17.32:8088', '192.168.127.12:9188', '192.168.3.11:23668', '172.16.17.32:3537', '192.168.3.11:1080', '172.16.17.32:9051', '172.16.17.32:8050', '192.168.3.11:1080', '172.16.58.3:10801', '192.168.127.12:45668', '192.168.3.11:48540', '192.168.3.11:10801', '172.16.58.3:34568', '192.168.127.12:14921', '192.168.3.11:1080', '192.168.127.12:10138', '192.168.127.12:1080', '172.16.17.32:15294', '172.16.31.10:50003', '192.168.3.11:1080', '192.168.127.12:59641', '192.168.3.11:51591', '192.168.127.12:63669', '192.168.3.11:6111', '192.168.127.12:63321', '172.16.31.10:6111', '172.16.58.3:48540', '172.16.58.3:1080', '172.16.58.3:1080', '192.168.127.12:1645', '172.16.17.32:57728', '192.168.3.11:12345', '172.16.31.10:4145', '192.168.3.11:1080', '192.168.3.11:6111', '172.16.31.10:1080', '172.16.17.32:9520', '172.16.17.32:30588', '192.168.3.11:1080', '172.16.58.3:8083', '192.168.3.11:1080', '172.16.17.32:1080', '192.168.127.12:31034', '192.168.3.11:1080', '192.168.127.12:1080', '192.168.127.12:9219', '172.16.31.10:6111', '192.168.127.12:1080', '172.16.58.3:1080', '172.16.58.3:40801', '172.16.58.3:25503', '172.16.17.32:53763', '172.16.17.32:6667', '192.168.3.11:9000', '172.16.58.3:1101', '172.16.17.32:1080', '172.16.17.32:22413', '172.16.17.32:9059', '192.168.127.12:55034', '192.168.127.12:1080', '172.16.31.10:1080', '172.16.17.32:9925', '192.168.127.12:61324', '172.16.17.32:9999', '192.168.3.11:1080', '192.168.3.11:31337', '172.16.58.3:1080', '172.16.17.32:28388', '172.16.58.3:6667', '172.16.17.32:1080', '192.168.127.12:4145', '172.16.31.10:58163', '172.16.17.32:6111', '172.16.17.32:7777', '172.16.17.32:3582', '172.16.58.3:10104', '172.16.17.32:1080', '172.16.17.32:16137', '172.16.31.10:443', '192.168.127.12:53763', '172.16.31.10:50015', '172.16.58.3:4145', '172.16.31.10:1437', '192.168.127.12:38844', '172.16.31.10:48540', '172.16.31.10:7529', '172.16.31.10:48550', '172.16.17.32:52441', '172.16.31.10:1080', '172.16.31.10:6111', '172.16.31.10:48540', '172.16.31.10:1080', '172.16.17.32:58890', '192.168.3.11:48540', '192.168.3.11:48540', '192.168.127.12:6111', '172.16.17.32:4145', '172.16.58.3:19404', '172.16.58.3:4492', '192.168.3.11:1080', '172.16.17.32:6111', '172.16.31.10:18762', '172.16.17.32:3558', '172.16.17.32:62625', '172.16.31.10:10555', '172.16.17.32:1080', '172.16.17.32:16379', '192.168.3.11:49538', '172.16.17.32:7529', '192.168.127.12:1080', '172.16.17.32:8080', '172.16.31.10:1080', '172.16.58.3:1080', '172.16.58.3:9090', '172.16.58.3:1080', '172.16.17.32:30588', '172.16.31.10:6111', '192.168.127.12:25503', '192.168.127.12:9054', '172.16.58.3:1080', '172.16.58.3:1080', '172.16.17.32:31353', '192.168.3.11:10086', '172.16.58.3:1080', '172.16.31.10:6111', '172.16.31.10:1059', '192.168.3.11:6111', '172.16.31.10:12563', '172.16.17.32:58208', '192.168.127.12:30588', '172.16.17.32:8888', '172.16.31.10:50013', '172.16.58.3:9090', '192.168.3.11:1080', '192.168.127.12:10808', '192.168.127.12:9469', '172.16.31.10:9050', '172.16.17.32:3553', '172.16.31.10:3128', '192.168.3.11:6111', '172.16.31.10:1080', '172.16.31.10:10581', '172.16.17.32:3595', '172.16.58.3:4145', '192.168.127.12:26236', '172.16.31.10:1080', '172.16.17.32:1080', '172.16.58.3:1080', '172.16.17.32:3517', '172.16.17.32:9050', '172.16.17.32:6667', '172.16.17.32:53468', '172.16.58.3:4145', '172.16.58.3:1080', '192.168.3.11:1085', '192.168.3.11:6666', '172.16.31.10:4145', '172.16.58.3:1080', '172.16.17.32:1080', '192.168.3.11:1080', '172.16.58.3:1080', '192.168.127.12:9102', '192.168.127.12:1080', '172.16.31.10:4145', '172.16.31.10:1645', '172.16.31.10:50045', '172.16.31.10:8293', '192.168.3.11:3128', '192.168.127.12:57529', '192.168.127.12:18301', '172.16.17.32:1080', '172.16.31.10:1080', '172.16.58.3:8090', '192.168.127.12:24260', '172.16.31.10:6111', '172.16.31.10:6111', '172.16.17.32:32389', '192.168.127.12:8000', '192.168.127.12:63868', '192.168.127.12:19402', '172.16.17.32:4492', '192.168.127.12:10086', '172.16.17.32:6667', '192.168.127.12:6111', '192.168.3.11:1080', '172.16.17.32:3568', '172.16.58.3:1080', '172.16.31.10:4145', '192.168.127.12:58163', '192.168.127.12:58890', '192.168.3.11:1080', '192.168.3.11:9300', '172.16.31.10:8888', '172.16.17.32:36916', '172.16.58.3:1080', '192.168.3.11:1080', '192.168.127.12:10808', '192.168.127.12:1080', '192.168.127.12:6111', '172.16.58.3:9999', '192.168.127.12:20052', '172.16.58.3:8080', '172.16.31.10:6111', '172.16.31.10:6111', '172.16.58.3:6111', '172.16.58.3:4145', '172.16.31.10:9999', '172.16.58.3:1080', '192.168.3.11:30588', '172.16.31.10:8888', '172.16.31.10:9051', '172.16.17.32:1080', '192.168.127.12:6111', '192.168.3.11:10086', '192.168.3.11:80', '172.16.17.32:30588', '172.16.31.10:10366', '172.16.17.32:3592', '172.16.17.32:6111', '172.16.31.10:1080', '172.16.58.3:1080', '172.16.17.32:10086', '172.16.17.32:1080', '192.168.127.12:50725', '192.168.127.12:12345', '172.16.31.10:9050', '192.168.127.12:47150', '172.16.58.3:9300', '192.168.3.11:1080', '192.168.127.12:1080', '172.16.31.10:4145', '192.168.3.11:25283', '172.16.17.32:4145', '172.16.17.32:6667', '192.168.127.12:1080', '172.16.58.3:1080', '172.16.17.32:9897', '172.16.58.3:48540', '172.16.17.32:10843', '192.168.127.12:59874', '172.16.17.32:6111', '172.16.17.32:1081', '172.16.31.10:27229', '192.168.127.12:6111', '172.16.58.3:1080', '172.16.31.10:6111', '192.168.3.11:6111', '172.16.58.3:6667', '192.168.127.12:62625', '172.16.31.10:43778', '172.16.17.32:9682', '172.16.31.10:8081', '172.16.31.10:44771', '192.168.3.11:1080', '192.168.127.12:4145', '172.16.58.3:1083', '172.16.17.32:9712', '172.16.31.10:2067', '192.168.127.12:6111', '172.16.58.3:4145', '192.168.127.12:9274', '192.168.127.12:6111', '172.16.58.3:1080', '192.168.3.11:1080', '172.16.31.10:53763', '172.16.31.10:23779', '172.16.17.32:6111', '192.168.127.12:6111', '172.16.17.32:1080', '172.16.31.10:9999', '192.168.3.11:5678', '192.168.127.12:7158', '172.16.31.10:58163', '172.16.58.3:10738', '172.16.31.10:6667', '172.16.17.32:4145', '192.168.127.12:1645', '192.168.3.11:15268', '192.168.3.11:6111', '192.168.127.12:58890', '172.16.17.32:34424', '172.16.58.3:34454', '192.168.127.12:1080', '192.168.127.12:1080', '172.16.58.3:1084', '192.168.3.11:1080', '172.16.58.3:1080', '172.16.17.32:9739', '192.168.3.11:1080', '172.16.58.3:59874', '172.16.58.3:6111', '172.16.31.10:80', '172.16.31.10:1080', '172.16.31.10:1080', '192.168.127.12:19828', '172.16.31.10:1080', '192.168.3.11:4145', '172.16.17.32:35904', '172.16.17.32:20790', '172.16.17.32:3505', '192.168.127.12:6111', '172.16.17.32:31679', '172.16.17.32:28388', '172.16.31.10:4145', '172.16.58.3:10086', '192.168.3.11:1080', '172.16.17.32:3000', '172.16.31.10:15864', '172.16.31.10:1080', '172.16.58.3:1080', '192.168.3.11:1080', '192.168.127.12:4145', '192.168.3.11:6111', '172.16.58.3:6111', '192.168.127.12:1080', '192.168.3.11:8080', '192.168.3.11:6111', '172.16.17.32:34477', '172.16.58.3:3000', '192.168.3.11:1080', '172.16.17.32:9516', '172.16.17.32:6111', '172.16.58.3:1080', '192.168.127.12:18302', '172.16.17.32:6111', '172.16.31.10:1080', '172.16.31.10:1080', '192.168.127.12:5432', '172.16.58.3:6111', '172.16.17.32:6111', '172.16.58.3:10062', '172.16.58.3:3000', '192.168.3.11:1080', '172.16.17.32:1080', '172.16.31.10:8023', '192.168.127.12:51656', '172.16.17.32:8528', '172.16.31.10:1080', '192.168.127.12:1088', '172.16.58.3:6111', '172.16.17.32:3508', '172.16.31.10:9050', '172.16.17.32:15076', '192.168.127.12:1080', '172.16.17.32:44783', '172.16.17.32:1080', '172.16.58.3:12345', '192.168.3.11:17173', '192.168.127.12:1080', '172.16.31.10:38888', '192.168.3.11:1080', '172.16.58.3:27931', '192.168.3.11:1080', '172.16.17.32:50001', '172.16.31.10:1080', '172.16.31.10:50041', '172.16.17.32:8522', '172.16.17.32:22676', '192.168.127.12:9603', '192.168.3.11:4145', '192.168.3.11:1080', '172.16.17.32:53468', '172.16.17.32:3548', '192.168.3.11:57114', '172.16.58.3:1080', '172.16.17.32:46164', '172.16.17.32:6111', '192.168.127.12:64386', '192.168.3.11:1088', '192.168.127.12:48540', '192.168.3.11:6667', '192.168.127.12:6111', '172.16.31.10:1080', '172.16.17.32:1080', '192.168.127.12:8888', '192.168.127.12:1080', '172.16.31.10:1080', '172.16.31.10:9050', '192.168.127.12:9052', '172.16.17.32:9052', '172.16.31.10:8080', '172.16.17.32:18204', '192.168.127.12:31653', '172.16.17.32:59874', '192.168.127.12:9632', '172.16.17.32:8529', '192.168.127.12:6776', '192.168.3.11:5588', '192.168.127.12:4145', '172.16.17.32:31337', '172.16.58.3:1080', '172.16.17.32:1080', '172.16.31.10:50031', '192.168.127.12:1080', '172.16.17.32:37884', '172.16.17.32:10808', '192.168.3.11:1080', '192.168.127.12:4145', '172.16.17.32:1080', '192.168.127.12:1080', '172.16.58.3:4145', '172.16.17.32:9921', '192.168.3.11:25881', '172.16.31.10:62625', '192.168.127.12:64064', '172.16.31.10:6667', '172.16.58.3:1718', '192.168.3.11:16092', '192.168.127.12:1080', '192.168.127.12:4145', '172.16.31.10:4145', '172.16.58.3:10558', '192.168.127.12:6111', '172.16.17.32:10000', '172.16.31.10:6111', '192.168.127.12:28388', '172.16.17.32:6111', '192.168.127.12:1080', '192.168.3.11:4145', '192.168.3.11:36181', '172.16.58.3:1080', '172.16.58.3:9050', '172.16.31.10:1080', '172.16.58.3:8080', '192.168.3.11:59435', '172.16.58.3:1645', '172.16.58.3:6111', '172.16.58.3:10278', '192.168.3.11:6111', '172.16.17.32:6667', '172.16.17.32:1080', '172.16.58.3:8083', '192.168.127.12:58163', '192.168.3.11:9050', '172.16.58.3:1454', '172.16.17.32:3584', '172.16.17.32:3530', '172.16.58.3:1080', '172.16.31.10:38844', '192.168.127.12:5678', '172.16.17.32:53763', '192.168.3.11:6667', '192.168.3.11:1080', '172.16.58.3:6667', '172.16.58.3:6111', '172.16.17.32:3509', '172.16.31.10:37884', '172.16.17.32:6111', '172.16.17.32:1081', '172.16.58.3:9999', '192.168.127.12:1080', '192.168.127.12:1080', '172.16.17.32:6111', '192.168.127.12:2364', '172.16.31.10:10086', '192.168.127.12:60966', '192.168.3.11:1088', '172.16.58.3:1080', '172.16.31.10:9050', '172.16.17.32:9968', '192.168.127.12:9230', '172.16.17.32:3519', '192.168.3.11:10808', '192.168.127.12:35828', '192.168.127.12:8888', '192.168.127.12:28388', '192.168.127.12:6667', '172.16.17.32:9999', '172.16.17.32:9050', '172.16.58.3:42461', '172.16.17.32:36404', '172.16.31.10:6111', '172.16.17.32:38888', '172.16.31.10:1645', '172.16.58.3:1080', '172.16.58.3:6667', '192.168.127.12:9150', '192.168.3.11:3128', '172.16.58.3:1080', '172.16.58.3:10919', '192.168.3.11:18321', '192.168.3.11:6111', '192.168.127.12:1080', '172.16.17.32:6111', '192.168.127.12:60757', '192.168.127.12:62625', '192.168.3.11:37607', '192.168.127.12:9000', '172.16.58.3:1080', '172.16.17.32:9000', '192.168.3.11:6667', '192.168.127.12:9451', '192.168.3.11:1080', '172.16.58.3:5678', '172.16.17.32:1234', '172.16.58.3:8081', '192.168.127.12:8081', '172.16.31.10:1080', '172.16.17.32:1081', '192.168.127.12:8081', '172.16.17.32:8081', '172.16.31.10:8081', '172.16.17.32:3520', '172.16.58.3:1080']
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 OPR/78.0.4093.153","Accept-Encoding":"gzip, deflate", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
    for proxy in proxylist:
        page = requests.get(url,headers=headers,proxies={'http://':proxy,'https://':proxy},timeout=2)
        if page.status_code==200:
            break
        else:
            continue
    soup1 = BeautifulSoup(page.content,"html.parser")
    soup2 = BeautifulSoup(soup1.prettify(), "html.parser")
    return soup2<jupyter_output><empty_output><jupyter_text>Getting the product data attributes<jupyter_code>def get_data(soup):
    product_data_list = []
    product_data = soup.find_all('div',{'data-component-type':'s-search-result'})
    product_data_dict = {}
    for item in product_data:
        flag=0
        try:
            product_data_dict = {
            'Product_Name':item.find(class_="a-size-medium a-color-base a-text-normal").get_text().strip(),
            'Product_Price': float(item.find(class_="a-price-whole").get_text().strip().replace(',','').replace('₹','')),
            'Actual_Product_Price': float(((item.find(class_="a-price a-text-price")).find(class_="a-offscreen")).get_text().strip().replace(' ','').replace('\n','').replace(',','').replace('₹','')),
            'Product_Rating(5)': float(item.find(class_="a-icon-alt").get_text().strip().strip(" out of 5 stars")),
            'No._of_ratings': int(item.find(class_="a-size-base").get_text().strip().replace(',','')),
            'Link': item.find(class_="a-link-normal s-link-style")['href'].strip(),
            }
        except:
            flag = 1
            pass
        if flag==0:
            product_data_list.append(product_data_dict)
        else:
            flag=0
            continue
    return product_data_list<jupyter_output><empty_output><jupyter_text>Testing the get_soup() function<jupyter_code>soup = get_soup("https://www.amazon.in/s?k=laptops&page=2&ref=sr_pg_2")
soup<jupyter_output><empty_output><jupyter_text>Extracting the product attributes from the extracted soup using get_data() function<jupyter_code>product = get_data(soup)
len(product)<jupyter_output><empty_output><jupyter_text>Making a function for checking out the rate of extraction<jupyter_code>def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()<jupyter_output><empty_output><jupyter_text>Checking out the final implementation<jupyter_code>def product_data(keyword):
    product_data_list = []
    product_data_list_final = []

    printProgressBar(0,20,prefix = 'Progress:', suffix = 'Complete', length = 50)
    x = 1
    while True:
        url = 'https://www.amazon.in/s?k='+keyword+'&page='+str(x)+'&ref=sr_pg_'+str(x)
        try:
            soup_var = get_soup(url)
            product_data_list = get_data(soup_var)
            if product_data_list==[]:
                continue
        except:
            continue
        product_data_list_final.extend(product_data_list)
        #print(len(product_data_list_final))
        time.sleep(1.0)
        printProgressBar(x,20,prefix = 'Progress:', suffix = 'Complete', length = 50)
        x = x + 1
        if x==21:
            break
        if not soup_var.find('li',{'class':'a-disabled a-last'}):
            pass
        else:
            break
    df = pd.DataFrame(product_data_list_final)
    return df
df = product_data("laptops")<jupyter_output>Progress: |██████████████████████████████████████████████████| 100.0% Complete<jupyter_text>The final dataframe<jupyter_code>df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.27726191824255464, 'fraction_numerical': 0.4553278224436644, 'mean_word_length': 4.03076046710339, 'pattern_counts': {'":': 6, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 586, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '124702', 'license': 'MIT', 'n_tokens_mistral': 13188, 'n_tokens_neox': 8510, 'n_words': 886, 'path': 'Testing the Products Data Attributes.ipynb', 'repo_name': 'subhajit2001/Making-of-AmazonProductScrapper-Package', 'size': '18207'}