Spaces:
Sleeping
Sleeping
File size: 1,447 Bytes
8ef7527 046cc67 8ef7527 aa42935 8ef7527 152aad4 8ef7527 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
from selenium import webdriver
from selenium.webdriver.common.by import By
import streamlit as st
from selenium.webdriver import FirefoxOptions
import re
import logging
def scrape_text(url, n_words=15):
try:
driver = None
logging.warning("Initiated Scraping")
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
opts = FirefoxOptions()
opts.add_argument("--headless")
opts.add_argument(f"user-agent={user_agent}")
driver = webdriver.Firefox(options=opts)
driver.set_page_load_timeout(60)
driver.get(url)
elem = driver.find_element(By.TAG_NAME, "body").text
# h1 = driver.find_element(By.TAG_NAME, "h1").text
# elem = h1 + "\n" + elem
sents = elem.split("\n")
sentence_list = []
for sent in sents:
sent = sent.strip()
if (len(sent.split()) >= n_words) and (len(re.findall(r"^\w.+[^\w\)\s]$", sent))>0):
sentence_list.append(sent)
driver.quit()
logging.warning("Closed Webdriver")
logging.warning("Successfully scraped text")
if len(sentence_list) < 3:
raise Exception("Found nothing to scrape.")
return " \n\n\n".join(sentence_list)
except:
if driver:
driver.close()
logging.warning("Closed Webdriver")
raise
|