<commit_before>import requests
import logging
import signal
import time
from selenium import webdriver

logger = logging.getLogger('webPageLoader')

def getHtmlStatic(url, max_retries = 2):
    nRetries = 0
    while (True):
        try:
            return requests.get(url, timeout = 10).text
        except Exception as e:
            if (nRetries >= max_retries):
                raise e
            else:
                time.sleep(10)
                nRetries = nRetries + 1

def loadPageAndGetHtml(url):
  pageHtml = ""

  try:
    driver = webdriver.PhantomJS(executable_path='node_modules/phantomjs-prebuilt/bin/phantomjs')
    driver.implicitly_wait(30)
    driver.set_page_load_timeout(30)

    try:
      driver.get(url)
      pageHtml = driver.page_source
    finally:
      try:
        driver.service.process.send_signal(signal.SIGTERM)
        driver.quit()
      except:
        pass
  except:
    logger.info("Could not load page with url %s through selenium", url)
    pass

  if not pageHtml:
    pageHtml = getHtmlStatic(url)

  return pageHtml
<commit_msg>pass userAgent header while getting pages statically<commit_after>import requests
import logging
import signal
import time
from selenium import webdriver

logger = logging.getLogger('webPageLoader')

def getHtmlStatic(url, max_retries = 2):
    nRetries = 0
    headers = {'User-Agent': 'newsbot'}
    while (True):
        try:
            return requests.get(url, headers = headers, timeout = 10).text
        except Exception as e:
            if (nRetries >= max_retries):
                raise e
            else:
                time.sleep(10)
                nRetries = nRetries + 1

def loadPageAndGetHtml(url):
  pageHtml = ""

  try:
    driver = webdriver.PhantomJS(executable_path='node_modules/phantomjs-prebuilt/bin/phantomjs')
    driver.implicitly_wait(30)
    driver.set_page_load_timeout(30)

    try:
      driver.get(url)
      pageHtml = driver.page_source
    finally:
      try:
        driver.service.process.send_signal(signal.SIGTERM)
        driver.quit()
      except:
        pass
  except:
    logger.info("Could not load page with url %s through selenium", url)
    pass

  if not pageHtml:
    pageHtml = getHtmlStatic(url)

  return pageHtml

{'fraction_non_alphanumeric': 0.08570159857904085, 'fraction_numerical': 0.009769094138543518, 'mean_word_length': 2.482225656877898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2725970', 'n_tokens_mistral': 715, 'n_tokens_neox': 669, 'n_words': 167}