<jupyter_start><jupyter_text>Libraries<jupyter_code>#Import All Dependencies
# import cv2, os, bz2, json, csv, difflib, requests, socket, whois, urllib.request, urllib.parse, urllib.error, re, OpenSSL, ssl
import numpy as np
from datetime import datetime
from urllib.parse import urlparse
from urllib.request import Request, urlopen
# from selenium import webdriver
from matplotlib import pyplot as plt
from bs4 import BeautifulSoup
# from timeout import timeout
import requests
import numpy as np
import urllib
import cv2
import re
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.firefox.options import Options
from PIL import Image
from io import BytesIO
import time
import os
import os.path
from os import path
import io
from difflib import SequenceMatcher
import contextlib 
  
try: 
    from urllib.parse import urlencode           
except ImportError: 
    from urllib import urlencode 
try: 
    from urllib.request import urlopen 
except ImportError: 
    from urllib2 import urlopen 

import sys<jupyter_output><empty_output><jupyter_text>TinyURL<jupyter_code>#Taken from https://www.geeksforgeeks.org/python-url-shortener-using-tinyurl-api/
#Returns the url subtracting the domain name with www and com stuff
#So, http://tinyurl.com/y5bffkh2 ---becomes---> y5bffkh2
def getTinyURL(URL): 
    request_url = ('http://tinyurl.com/api-create.php?' + urlencode({'url':URL}))     
    with contextlib.closing(urlopen(request_url)) as response:                       
        return response.read().decode('utf-8 ')[19:]
#Returns Beautiful Soup object
def getHTML(URL):
    try:
        hdr = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'} #Make the user agent verified, that is Mozilla
#         req = Request(URL,headers=hdr)
        req = requests.get(URL, headers=hdr)
        page = req.text #Get URL HTML contents
        soup = BeautifulSoup(page, 'html.parser') #Convert to BeutifulSoup
        print("Built Soup")
        #prettyText = str(soup.prettify()) #Convert the HTML in its form
        return soup
    except Exception as e:
# #         if e.__class__.__name__ == "TimeoutError": raise TimeoutError("")
        return None<jupyter_output><empty_output><jupyter_text>Selenium<jupyter_code>fireFoxBinaryPath = r'C:\Program Files (x86)\Mozilla Firefox\firefox.exe'
geckoDriverPath = "C:/Users/Yash/Documents/geckodriver.exe"

googleChomreDriverPath = r"C:\Users\Yash\Documents\chromedriver.exe"

windowSizeXAxis = 1024
windowSizeYAxis = 768
# Uses Firefox to launch Selenium Headlessly: One needs to have:
# firefox binary path(located in Program Files folder(Windows)) and 
# geckoDriver, which acts as an intermediate between selenium and firefox, and can download it from: https://github.com/mozilla/geckodriver/releases
def getSeleniumDriverForFirefox(fireFoxBinaryPath, geckoDriverPath, windowSizeXAxis, windowSizeYAxis):
    options = Options()
    options.set_headless(headless=True)
    options.binary = binary
    options.add_argument('--ignore-certificate-errors')
    
    cap = DesiredCapabilities().FIREFOX
    cap["marionette"] = True #optional
    
    driver = webdriver.Firefox(firefox_options=options, capabilities=cap, executable_path=geckodriver)
    driver.set_window_size(windowSizeXAxis, windowSizeYAxis) # set the window size that you need 
    return driver

def getSeleniumDriverForChrome(googleChomreDriverPath):
    options = webdriver.ChromeOptions()
    options.headless = False

    driver = webdriver.Chrome(options=options, executable_path=googleChomreDriverPath)
    return driver

# Saves the screenshot with the last 20 characters of the URL as the file name
# Firefox doesn't properly returns the whole page's screenshot, so use Chrome
def takeScreenshotWithFirefox(URL):
    driver2 = getSeleniumDriverForChrome(googleChomreDriverPath)
    driver2.get(URL)
    driver2.save_screenshot("screenshots/" + getTinyURL(URL)+".png")
    driver2.quit()
    
def takeScreenshotWithChrome(URL, driver):
    driver.get(URL)

    S = lambda X: driver.execute_script('return document.body.parentNode.scroll'+X)
    driver.set_window_size(S('Width'),S('Height')) # May need manual adjustment
    driver.find_element_by_tag_name('body').screenshot("screenshots/" + getTinyURL(URL)+".png")<jupyter_output><empty_output><jupyter_text>File Handling<jupyter_code># Create target directory & all intermediate directories if don't exists
def createdDirectory(path):
    try:
        os.mkdir(path)    
        print("Directory " , path ,  " Created ")
    except FileExistsError:
        print("Directory " , path ,  " already exists")<jupyter_output><empty_output><jupyter_text>DOM<jupyter_code>#https://selenium-python.readthedocs.io/locating-elements.html#locating-by-xpath
#XPath can either id or name relative
def getXPath(currentTag):
    returnXPath = ""
    while(currentTag.parent!=None):
        if "id" in (currentTag.attrs):
            print("true")
            returnXPath = currentTag.name + "[@id='" + currentTag.attrs['id'].strip() + "']/" + returnXPath #//form[@id='loginForm']
            break
        if "name" in (currentTag.attrs):
            print("true")
            returnXPath = currentTag.name + "[@name='" + currentTag.attrs['name'].strip() + "']/" + returnXPath #//form[@id='loginForm']
            break 
        returnXPath = currentTag.name + "/" + returnXPath
        currentTag = currentTag.parent
        print(currentTag.attrs)
        
        
    returnXPath = returnXPath.replace("[document]/html","/") #When it reaches the end of document parent, it adds the following it it's start, so we need to delete it
    if not returnXPath.startswith("//"): #the XPath should start with 2 forward slash
        returnXPath = "//" + returnXPath
    if returnXPath.endswith("/"): #the XPath should not end with forward slash
        returnXPath = returnXPath[:-1] 
    returnXPath = returnXPath.replace("meta/","").replace("table/tr", "table/tbody/tr") #Few changes to be made while performing XPath
    return (returnXPath) #//div[@id='tab-panel-0-w3']/div/span/h2<jupyter_output><empty_output><jupyter_text>Price<jupyter_code>def getTheTagElementForPrice(gShopPriceUpdated, soup):
    returnPriceElementTag = None
    try:
        dummyVar = soup(text=re.compile(gShopPriceUpdated))
    #     print(dummyVar)
        for elem in dummyVar:
#             print("elem.parent",elem.parent)
#             print("elem.parent.name",elem.parent.name)
            if returnPriceElementTag == None: #The first element is the return value unless we encounter a heading tag
                returnPriceElementTag = elem.parent
            if "h" in elem.parent.name: #Found the heading tag, so return this tag and break the loop
                returnPriceElementTag = elem.parent
                return returnPriceElementTag
    except Exception as e:
        print("Error in getTheTagElementForPrice(gShopPriceUpdated, soup)")
    return returnPriceElementTag

def findPriceElementTag(gShopPrice, soup): #gShopPrice = $379.00
    gShopPrice = gShopPrice.replace("now","").strip() #GShop soemtimes gives prices with now suffix like "$0.00 now"
    print(gShopPrice)
    
    if "$" in gShopPrice:
        gShopPriceUpdated = gShopPrice.replace("$", "\$") #gShopPriceUpdated = \$379.00; Required because $ is reserved keyword for regex
        print(gShopPriceUpdated)
        returnPriceElementTag = getTheTagElementForPrice(gShopPriceUpdated, soup)
        if returnPriceElementTag != None and len(str(returnPriceElementTag))<400:
            return returnPriceElementTag
    
    gShopPriceUpdated = gShopPrice.replace("$","") #gShopPriceUpdated = 379.00
    print(gShopPriceUpdated)
    returnPriceElementTag = getTheTagElementForPrice(gShopPriceUpdated, soup)
    if returnPriceElementTag != None and len(str(returnPriceElementTag))<400:
        return returnPriceElementTag
        
    gShopPriceUpdated = gShopPrice.replace("$", "\$").split(".")[0] #gShopPriceUpdated = \$379
    print(gShopPriceUpdated)
    returnPriceElementTag = getTheTagElementForPrice(gShopPriceUpdated, soup)
    if returnPriceElementTag != None and len(str(returnPriceElementTag))<400:
        return returnPriceElementTag
        
    gShopPriceUpdated = gShopPrice.replace("$","").split(".")[0] #gShopPriceUpdated = 379
    print(gShopPriceUpdated)
    returnPriceElementTag = getTheTagElementForPrice(gShopPriceUpdated, soup)
    if returnPriceElementTag != None and len(str(returnPriceElementTag))<400:
        return returnPriceElementTag
        
    return None<jupyter_output><empty_output><jupyter_text>Heading<jupyter_code>def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()
def findHeadingElementTag(gShopPrice, soup):
    maxSimilarRatio = 0
    resultHeadingElementTag = None

    allHeadingTags = soup.find_all(re.compile(r'h\d+'))
    for headingTag in allHeadingTags:
    #     print(headingTag)
        if headingTag.string!=None:
            if maxSimilarRatio<similar(gShopPrice, headingTag.string):
                maxSimilarRatio = similar(gShopPrice, headingTag.string)
                resultHeadingElementTag = headingTag
    #Threshold
    if maxSimilarRatio<0.2:
        allHeadingTags = soup.find_all(re.compile(r'span'))
        for headingTag in allHeadingTags:
        #     print(headingTag)
            if headingTag.string!=None:
                if maxSimilarRatio<similar(gShopPrice, headingTag.string):
                    maxSimilarRatio = similar(gShopPrice, headingTag.string)
                    resultHeadingElementTag = headingTag
                
    print("resultHeadingElementTag:",resultHeadingElementTag)
    print("maxSimilarRatio:",maxSimilarRatio)
    
    return resultHeadingElementTag<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>GShopBasePath = r"C:\Users\Yash\Desktop\UIUC_RA\GoogleShoppingScraper\Verson 3\GShop"
listOfGShopProducts = os.listdir(GShopBasePath)
listOfGShopProducts
for GShopProducts in listOfGShopProducts:
    datalogsPath = GShopBasePath + "\\" + GShopProducts + "\datalogs"
    listOfDatalogs = os.listdir(datalogsPath)
    screenshotsImagesPath = GShopBasePath + "\\" + GShopProducts + "\screenshots"
    listOfScreenshots = os.listdir(screenshotsImagesPath)
    
    for screenshotImagesNames in listOfScreenshots: #Make bounding box for each image
        
        #There can be cases where the screenshot was successful, but price could not be obtained
        if not path.exists(datalogsPath + "/" + screenshotImagesNames[:-4] + ".txt"):
            continue
        
        #Get the URL of current screenshot
        print("http://tinyurl.com/" + screenshotImagesNames[:-4])
        URLOfCurrentScreenshot = "http://tinyurl.com/" + screenshotImagesNames[:-4]
        
        #Get the Price of current screenshot
        file = open(datalogsPath + "/" + screenshotImagesNames[:-4] + ".txt", "r", encoding='utf-8') 
        priceOfCurrentScreenshot = str(file.read()).split(" ,;, ")[1]
        print("priceOfCurrentScreenshot:",priceOfCurrentScreenshot)
        
        #Get the Heading of current screenshot
        file = open(datalogsPath + "/" + screenshotImagesNames[:-4] + ".txt", "r", encoding='utf-8') 
        headingeOfCurrentScreenshot = str(file.read()).split(" ,;, ")[0].strip()
        print("headingeOfCurrentScreenshot:",headingeOfCurrentScreenshot)
        
        #Find Price Element
        soup = getHTML(URLOfCurrentScreenshot)
        returnPriceElementTag = findPriceElementTag(priceOfCurrentScreenshot, soup)
        print(returnPriceElementTag)
        print(len(str(returnPriceElementTag)))
        
        #Find Heading Element
        returnHeadingElementTag = findHeadingElementTag(headingeOfCurrentScreenshot, soup)
        print(returnHeadingElementTag)
        print(len(str(returnHeadingElementTag)))        
        
#         try:
        
        #If priceElement is not None, then find it's location and take it's cropped screenshot and save
        if returnPriceElementTag!=None:

            #Get XPath
            # For Price
            XPathForPriceElementTag = getXPath(returnPriceElementTag)
            print("XPathForPriceElementTag:",XPathForPriceElementTag) 
            # For Price
            XPathForHeadingElementTag = getXPath(returnHeadingElementTag)
            print("XPathForHeadingElementTag:",XPathForHeadingElementTag)


            #Get the first line of dataLog: Helps in rewriting the XPath
            f = open(datalogsPath + "/" + screenshotImagesNames[:-4] + ".txt","r", encoding='utf-8')
            first_line = f.readline()
            first_line = first_line.split("PriceXPath")[0]
            first_line = first_line.split("\n")[0]
            print("first_line:",first_line)
            f.close()
            
            #Reopen the dataLog freshly and rewrite the whole thing
            f = open(datalogsPath + "/" + screenshotImagesNames[:-4] + ".txt","w", encoding='utf-8')
            
            #Write the Heading ,;, Price
            f.write(first_line)
    
            #Write with "PriceXPath: <XPath>"
            f.write("\nPriceXPath: " + str(XPathForPriceElementTag))

            #Write with "HeadingXPath: <XPath>"
            f.write("\nHeadingXPath: " + str(XPathForHeadingElementTag))

            print("WROTEEEEEEEEEEEEEEEE")
            f.close()

#         except Exception as e:
#             print("Driver Failed")
#             print(e)<jupyter_output>http://tinyurl.com/y54a4wl9
priceOfCurrentScreenshot: $15.00/mo
headingeOfCurrentScreenshot: Samsung Galaxy S10 Plus - 512 GB - Ceramic White - Sprint - CDMA/GSM
Built Soup
$15.00/mo
\$15.00/mo
15.00/mo
\$15
15
None
4
resultHeadingElementTag: <h1 class="mt-0 mb-10 text-center" device-name="">Samsung Galaxy S10+</h1>
maxSimilarRatio: 0.41379310344827586
<h1 class="mt-0 mb-10 text-center" device-name="">Samsung Galaxy S10+</h1>
74
http://tinyurl.com/y6r8bxfz
priceOfCurrentScreenshot: $1,249.99
PriceXPath: /div/div/div/span/h2
HeadingXPath: /html/body/div[2]/div[3]/div[3]/div/div/div/h1
headingeOfCurrentScreenshot: Samsung Galaxy S10 Plus - 512 GB - Ceramic White - Sprint - CDMA/GSM
Built Soup
$1,249.99
PriceXPath: /div/div/div/span/h2
HeadingXPath: /html/body/div[2]/div[3]/div[3]/div/div/div/h1
\$1,249.99
PriceXPath: /div/div/div/span/h2
HeadingXPath: /html/body/div[2]/div[3]/div[3]/div/div/div/h1
1,249.99
PriceXPath: /div/div/div/span/h2
HeadingXPath: /html/body/div[2]/div[3]/div[3]/div[...]
{'fraction_non_alphanumeric': 0.10695187165775401, 'fraction_numerical': 0.015905662964486495, 'mean_word_length': 3.5035504785427602, 'pattern_counts': {'":': 1, '<': 45, '<?xml version=': 0, '>': 39, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '170162', 'license': 'MIT', 'n_tokens_mistral': 4679, 'n_tokens_neox': 4400, 'n_words': 981, 'path': 'WebInformationExtraction/.ipynb_checkpoints/ConfidenceBasedTIPRetrieval-checkpoint.ipynb', 'repo_name': 'yashsaboo/RA-at-FORWARD_Data_Lab', 'size': '14586'}