<commit_before>from selenium import webdriver

# TODO : read more on htmlunit
# http://docs.seleniumhq.org/docs/03_webdriver.jsp
# http://dionysus.uraganov.net/software/how-to-install-selenium-server-with-firefox-on-ubuntu-11-10/
# http://ubuntuforums.org/showthread.php?t=1894595

__author__ = 'grainier'
import contextlib
from selenium.webdriver import Firefox, FirefoxProfile, Remote
import time


class ApplicationIndexer(object):

    def __init__(self, url):
        self.url = url
        self.fp = FirefoxProfile()
        self.fp.set_preference('permissions.default.stylesheet', 2)                     # Disable css
        self.fp.set_preference('permissions.default.image', 2)                          # Disable images
        self.fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')    # Disable Flash
        pass

    def get_scraped_apps(self):
        applications = self.get_applications_in_page()
        return applications
        pass

    def get_applications_in_page(self):
        applications = []
        # with contextlib.closing(Firefox(firefox_profile=self.fp)) as driver:
        with contextlib.closing(webdriver.Remote("http://localhost:4444/wd/hub", webdriver.DesiredCapabilities.HTMLUNITWITHJS)) as driver:
            driver.get(self.url)
            driver.execute_script(
                "scraperLoadCompleted = false;" +
                "var interval = null, previousDocHeight = 0;" +
                "interval = setInterval(function () {" +
                "if (previousDocHeight < document.body.scrollHeight) {" +
                "window.scrollTo(0, Math.max(document.documentElement.scrollHeight," +
                "document.body.scrollHeight, document.documentElement.clientHeight));" +
                "document.getElementById('show-more-button').click();" +
                "previousDocHeight = document.body.scrollHeight;" +
                "} else {" +
                "clearInterval(interval);" +
                "scraperLoadCompleted = true;"
                "}" +
                "}, 4000);"
            )

            done = False
            while not done:
                # Wait for the script to complete
                time.sleep(2)
                done = driver.execute_script(
                    "return scraperLoadCompleted"
                )
                pass

            product_matrix = driver.find_elements_by_class_name("card")
            for application in product_matrix:
                applications.append(self.extract_application_data(application))
                pass
            pass<commit_msg>attempt with html unit driver<commit_after>from selenium import webdriver

# TODO : read more on htmlunit
# http://docs.seleniumhq.org/docs/03_webdriver.jsp
# http://dionysus.uraganov.net/software/how-to-install-selenium-server-with-firefox-on-ubuntu-11-10/
# http://ubuntuforums.org/showthread.php?t=1894595

__author__ = 'grainier'
import contextlib
from selenium.webdriver import Firefox, FirefoxProfile, Remote
import time


class ApplicationIndexer(object):

    def __init__(self, url):
        self.url = url
        self.fp = FirefoxProfile()
        self.fp.set_preference('permissions.default.stylesheet', 2)                     # Disable css
        self.fp.set_preference('permissions.default.image', 2)                          # Disable images
        self.fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')    # Disable Flash
        pass

    def get_scraped_apps(self):
        applications = self.get_applications_in_page()
        return applications
        pass

    def get_applications_in_page(self):
        applications = []
        # with contextlib.closing(Firefox(firefox_profile=self.fp)) as driver:
        with contextlib.closing(webdriver.Remote("http://localhost:4444/wd/hub", webdriver.DesiredCapabilities.FIREFOX)) as driver:
            driver.get(self.url)
            driver.execute_script(
                "scraperLoadCompleted = false;" +
                "var interval = null, previousDocHeight = 0;" +
                "interval = setInterval(function () {" +
                "if (previousDocHeight < document.body.scrollHeight) {" +
                "window.scrollTo(0, Math.max(document.documentElement.scrollHeight," +
                "document.body.scrollHeight, document.documentElement.clientHeight));" +
                "document.getElementById('show-more-button').click();" +
                "previousDocHeight = document.body.scrollHeight;" +
                "} else {" +
                "clearInterval(interval);" +
                "scraperLoadCompleted = true;"
                "}" +
                "}, 4000);"
            )

            done = False
            while not done:
                # Wait for the script to complete
                time.sleep(2)
                done = driver.execute_script(
                    "return scraperLoadCompleted"
                )
                pass

            product_matrix = driver.find_elements_by_class_name("card")
            for application in product_matrix:
                applications.append(self.extract_application_data(application))
                pass
            pass
{'fraction_non_alphanumeric': 0.1019819126419088, 'fraction_numerical': 0.010005772561092939, 'mean_word_length': 2.066666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947291', 'n_tokens_mistral': 1474, 'n_tokens_neox': 1354, 'n_words': 237}