Spaces:

ksvmuralidhar
/

text_rewriter

Build error

ksvmuralidhar commited on Apr 7

Commit

f6d006c

•

1 Parent(s): e71b6e8

Update scraper.py

Files changed (1) hide show

scraper.py CHANGED Viewed

@@ -11,8 +11,10 @@ def scrape_text(url, n_words=15):
     try:
         driver = None
         logging.warning("Initiated Scraping")
         opts = FirefoxOptions()
         opts.add_argument("--headless")
         driver = webdriver.Firefox(options=opts)
         driver.set_page_load_timeout(60)
         driver.get(url)

     try:
         driver = None
         logging.warning("Initiated Scraping")
+        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
         opts = FirefoxOptions()
         opts.add_argument("--headless")
+        opts.add_argument(f"user-agent={user_agent}")
         driver = webdriver.Firefox(options=opts)
         driver.set_page_load_timeout(60)
         driver.get(url)