<commit_before>import os
import sys
import time
import logging
import urllib
import urllib2
import cookielib
import json as simplejson
import socket
try:
  # I'll complain later if you really need this
  from selenium import webdriver
  from selenium import common
except ImportError:
  pass
# global socket timeout, seemed to help with a weird connection reset issue in appengine
socket.setdefaulttimeout(60)

logger = logging.getLogger('tst')

#-----------------------------------------------------------------------#

devel_host = os.environ["ACRE_FREEBASE_SITE_ADDR"]
devel_port = os.environ["ACRE_FREEBASE_SITE_ADDR_PORT"]

class QunitFetcher:

  def __init__(self, browser='chrome', selenium_rh=None):
    self.drivers = []
    logger.debug('selenium remote: %s' % selenium_rh)
    if browser == 'firefox':
      if selenium_rh:
        self.driver = webdriver.Remote(selenium_rh, 
          webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
      else:
        self.driver = webdriver.Firefox()
      self.browser = 'ff'
    elif browser == 'chrome':
      if selenium_rh:
        self.driver = webdriver.Remote(selenium_rh, 
          webdriver.common.desired_capabilities.DesiredCapabilities.CHROME)
      else:
        self.driver = webdriver.Chrome()
      self.browser = 'ch'
    else:
      print 'we dont currently support that browser name:', browser
      sys.exit(1)
    print 'browser: %s %s' % (\
    self.driver.desired_capabilities.get('browserName'),
    self.driver.desired_capabilities.get('version'))

  def quit(self):
    self.driver.quit()

  def cleanup(self):
    self.quit()
  def fetchtest(self, pack, module, test_url):
    driver = self.driver
    try:
      driver.get(test_url)
      # poll for the existence of an element
      driver.implicitly_wait(30)
      container=driver.find_element_by_id('qunit-testresult')
      # tests are done, all should be there
      driver.implicitly_wait(1)
      # test results are in an <ol>
      container=driver.find_element_by_id('qunit-tests')
    except common.exceptions.NoSuchElementException:
      body = driver.page_source
      # what a server unavailable looks like in various browsers
      if ('Unable to connect' in body or
      'is not available' in body):
        msg = 'server unavailable'
      else:
        msg = 'unknown response:', body[:500]
      tid = '%s/%s' % (pack, module)
      msg = '%s %s' % (test_url, msg)
      return [1, 1, 0, {tid:['ERROR', msg]}]
    # pretty sure some qunit tests ran, so get output
    testresults=container.find_elements_by_xpath('li')
    tests = 0
    failures = 0
    skips = 0
    results = {}
    for t in testresults:
      name = "%s/%s:%s" % (pack, module,
              t.find_element_by_class_name("test-name").text)
      passed = int(t.find_element_by_class_name("passed").text)
      failed = int(t.find_element_by_class_name("failed").text)
      # there can be many asserts, but I consider it one test
      tests += 1
      if failed > 0:
        failures += 1
        results[name] = ['FAIL', t.text.encode('utf-8')]
    return [tests, failures, 0, results]

class AcreFetcher:

  def __init__(self, username=None, password=<PASSWORD>, api_url=None):
    if not username: username = os.environ.get("FSTEST_USERNAME")
    if not password: password = <PASSWORD>("FSTEST_PASSWORD")
    if not api_url: api_url = os.environ.get("ACRE_METAWEB_BASE_ADDR")
    print api_url
    self.username=username
    self.password=password
    self.api_url=api_url
    #if username: self.login()<commit_msg>bump up timeout for all tests to complete<commit_after>import os
import sys
import time
import logging
import urllib
import urllib2
import cookielib
import json as simplejson
import socket
try:
  # I'll complain later if you really need this
  from selenium import webdriver
  from selenium import common
except ImportError:
  pass
# global socket timeout, seemed to help with a weird connection reset issue in appengine
socket.setdefaulttimeout(60)

logger = logging.getLogger('tst')

#-----------------------------------------------------------------------#

TEST_TIMEOUT = 120
devel_host = os.environ["ACRE_FREEBASE_SITE_ADDR"]
devel_port = os.environ["ACRE_FREEBASE_SITE_ADDR_PORT"]

class QunitFetcher:

  def __init__(self, browser='chrome', selenium_rh=None):
    self.drivers = []
    logger.debug('selenium remote: %s' % selenium_rh)
    if browser == 'firefox':
      if selenium_rh:
        self.driver = webdriver.Remote(selenium_rh, 
          webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX)
      else:
        self.driver = webdriver.Firefox()
      self.browser = 'ff'
    elif browser == 'chrome':
      if selenium_rh:
        self.driver = webdriver.Remote(selenium_rh, 
          webdriver.common.desired_capabilities.DesiredCapabilities.CHROME)
      else:
        self.driver = webdriver.Chrome()
      self.browser = 'ch'
    else:
      print 'we dont currently support that browser name:', browser
      sys.exit(1)
    print 'browser: %s %s' % (\
    self.driver.desired_capabilities.get('browserName'),
    self.driver.desired_capabilities.get('version'))

  def quit(self):
    self.driver.quit()

  def cleanup(self):
    self.quit()
  def fetchtest(self, pack, module, test_url):
    driver = self.driver
    try:
      driver.get(test_url)
      # this should show up right away
      container=driver.find_element_by_id('qunit-tests')
      # this signifies all tests are done
      driver.implicitly_wait(TEST_TIMEOUT)
      container=driver.find_element_by_id('qunit-testresult')
      driver.implicitly_wait(1)
    except common.exceptions.NoSuchElementException:
      body = driver.page_source
      # what a server unavailable looks like in various browsers
      if ('Unable to connect' in body or
      'is not available' in body):
        msg = 'server unavailable'
      else:
        msg = 'unknown response:', body[:500]
      tid = '%s/%s' % (pack, module)
      msg = '%s %s' % (test_url, msg)
      return [1, 1, 0, {tid:['ERROR', msg]}]
    # pretty sure some qunit tests ran, so get output
    container=driver.find_element_by_id('qunit-tests')
    testresults=container.find_elements_by_xpath('li')
    tests = 0
    failures = 0
    skips = 0
    results = {}
    for t in testresults:
      name = "%s/%s:%s" % (pack, module,
              t.find_element_by_class_name("test-name").text)
      passed = int(t.find_element_by_class_name("passed").text)
      failed = int(t.find_element_by_class_name("failed").text)
      # there can be many asserts, but I consider it one test
      tests += 1
      if failed > 0:
        failures += 1
        results[name] = ['FAIL', t.text.encode('utf-8')]
    return [tests, failures, 0, results]

class AcreFetcher:

  def __init__(self, username=None, password=<PASSWORD>, api_url=None):
    if not username: username = os.environ.get("FSTEST_USERNAME")
    if not password: password = os.environ.get("FSTEST_PASSWORD")
    if not api_url: api_url = os.environ.get("ACRE_METAWEB_BASE_ADDR")
    print api_url
    self.username=username
    self.password=password
    self.api_url=api_url
    #if username: self.login()
{'fraction_non_alphanumeric': 0.12437395659432388, 'fraction_numerical': 0.005982192543127434, 'mean_word_length': 2.9961089494163424, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1251028', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2224, 'n_words': 577}