Polo123's picture
Update steps/Bots.py
431ef39
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
from abc import ABC,abstractmethod
import pandas as pd
from steps.utils import get_priority_link
class BOT(ABC):
'''
Abstract method for selenium bot
'''
@abstractmethod
def target_html() -> str:
pass
class CompetitorBot(BOT):
'''
Get Competitor info for doing the steps in order:
1> google search company_name competitors
2> select most appropriate search result
3> Go to website, scrape
'''
def target_html(self,company_name: str) -> str:
browser = webdriver.Chrome(options=options)
#Searching on FireFox
browser.get('http://www.google.com')
search = browser.find_element("name", "q")
search.send_keys(company_name + " craft.co competitors")
search.send_keys(Keys.RETURN) # hit return after you enter search text
time.sleep(5)
# Getting Search results
search_results = browser.find_elements(By.XPATH,"//a")
links = [result.get_attribute("href") for result in search_results]
#Getting Priority Link
website,idx = get_priority_link(links)
print(website,idx)
print(links[idx])
#go to the website
search_results[idx].click()
time.sleep(10)
html = browser.page_source
browser.quit()
return website,html