<commit_before>/*
Copyright 2007-2010 Selenium committers

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */
package org.tribbloid.spookystuff.dsl

import com.gargoylesoftware.htmlunit.BrowserVersion
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkFiles
import org.openqa.selenium.htmlunit.HtmlUnitDriver
import org.openqa.selenium.phantomjs.{PhantomJSDriver, PhantomJSDriverService}
import org.openqa.selenium.remote.CapabilityType._
import org.openqa.selenium.remote.{BrowserType, CapabilityType, DesiredCapabilities}
import org.openqa.selenium.{Capabilities, Platform, Proxy}
import org.slf4j.LoggerFactory
import org.tribbloid.spookystuff.SpookyContext
import org.tribbloid.spookystuff.session.{CleanWebDriver, CleanWebDriverHelper, ProxySetting}

//TODO: switch to DriverPool! Tor cannot handle too many connection request.
sealed abstract class DriverFactory extends Serializable{

  final def newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {
    val result = _newInstance(capabilities, spooky)

    result
  }

  def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver
}

object DriverFactories {

  object PhantomJS {

    def phantomJSPath(fileName: String) = Option(System.getenv("PHANTOMJS_PATH"))
      .getOrElse{
      LoggerFactory.getLogger(this.getClass).info("$PHANTOMJS_PATH does not exist, using tempfile instead")
      SparkFiles.get(fileName)
    }

    //used in sc.addFile(...)
    val phantomJSUrl = System.getenv("PHANTOMJS_PATH") //TODO: download it from public resource
    val phantomJSFileName = new Path(phantomJSUrl).getName
  }

  case class PhantomJS(
                        fileName: String = PhantomJS.phantomJSFileName,
                        loadImages: Boolean = false
                        )
    extends DriverFactory {

    def exePath = PhantomJS.phantomJSPath(fileName)

    @transient lazy val baseCaps = new DesiredCapabilities(BrowserType.PHANTOMJS, "", Platform.ANY)
    baseCaps.setJavascriptEnabled(true); //< not really needed: JS enabled by default
    baseCaps.setCapability(CapabilityType.SUPPORTS_FINDING_BY_CSS, true)
    //  baseCaps.setCapability(CapabilityType.HAS_NATIVE_EVENTS, false)
    baseCaps.setCapability(TAKES_SCREENSHOT, true)
    baseCaps.setCapability(ACCEPT_SSL_CERTS, true)
    baseCaps.setCapability(SUPPORTS_ALERTS, true)
    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY, exePath)
    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "loadImages", loadImages)

    //    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX+"resourceTimeout", Const.resourceTimeout*1000)

    def newCap(capabilities: Capabilities, spooky: SpookyContext): DesiredCapabilities = {
      val result = new DesiredCapabilities(baseCaps)

      result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX+"resourceTimeout", spooky.conf.remoteResourceTimeout*1000)

      val userAgent = spooky.conf.userAgent
      if (userAgent != null) result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "userAgent", userAgent)

      val proxy = spooky.conf.proxy()

      if (proxy != null)
        result.setCapability(
          PhantomJSDriverService.PHANTOMJS_CLI_ARGS,
          Array("--proxy=" + proxy.addr+":"+proxy.port, "--proxy-type=" + proxy.protocol)
        )

      result.merge(capabilities)
    }

    //called from executors
    override def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {

      new PhantomJSDriver(newCap(capabilities, spooky)) with CleanWebDriverHelper
    }
  }

  case class HtmlUnit(
                       browser: BrowserVersion = BrowserVersion.FIREFOX_24
                       ) extends DriverFactory {

    val baseCaps = new DesiredCapabilities(BrowserType.HTMLUNIT, "", Platform.ANY)

    def newCap(capabilities: Capabilities, spooky: SpookyContext): DesiredCapabilities = {
      val result = new DesiredCapabilities(baseCaps)

      val userAgent = spooky.conf.userAgent
      //TODO: this is useless, need custom BrowserVersion
      //see http://stackoverflow.com/questions/12853715/setting-user-agent-for-htmlunitdriver-selenium
      if (userAgent != null) result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "userAgent", userAgent)

      val proxy: ProxySetting = spooky.conf.proxy()

      if (proxy != null) {
        result.setCapability(PROXY, proxy.toSeleniumProxy)
      }

      result.merge(capabilities)
    }

    override def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {

      val cap = newCap(capabilities, spooky)
      val driver = new HtmlUnitDriver(browser) with CleanWebDriverHelper
      driver.setJavascriptEnabled(true)
      driver.setProxySettings(Proxy.extractFrom(cap))

      driver
    }
  }

  ////just for debugging
  ////a bug in this driver has caused it unusable in Firefox 32
  //object FirefoxDriverFactory extends DriverFactory {
  //
  //  val baseCaps = new DesiredCapabilities
  //  //  baseCaps.setJavascriptEnabled(true);                //< not really needed: JS enabled by default
  //  //  baseCaps.setCapability(CapabilityType.SUPPORTS_FINDING_BY_CSS,true)
  //
  //  //  val FirefoxRootPath = "/usr/lib/phantomjs/"
  //  //  baseCaps.setCapability("webdriver.firefox.bin", "firefox");
  //  //  baseCaps.setCapability("webdriver.firefox.profile", "WebDriver");
  //
  //  override def newInstance(capabilities: Capabilities, spooky: SpookyContext): WebDriver = {
  //    val newCap = baseCaps.merge(capabilities)
  //
  //    Utils.retry(Const.DFSInPartitionRetry) {
  //      Utils.withDeadline(spooky.distributedResourceTimeout) {new FirefoxDriver(newCap)}
  //    }
  //  }
  //}

}<commit_msg>fix the same bug the third time<commit_after>/*
Copyright 2007-2010 Selenium committers

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
 */
package org.tribbloid.spookystuff.dsl

import com.gargoylesoftware.htmlunit.BrowserVersion
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkFiles
import org.openqa.selenium.htmlunit.HtmlUnitDriver
import org.openqa.selenium.phantomjs.{PhantomJSDriver, PhantomJSDriverService}
import org.openqa.selenium.remote.CapabilityType._
import org.openqa.selenium.remote.{BrowserType, CapabilityType, DesiredCapabilities}
import org.openqa.selenium.{Capabilities, Platform, Proxy}
import org.slf4j.LoggerFactory
import org.tribbloid.spookystuff.SpookyContext
import org.tribbloid.spookystuff.session.{CleanWebDriver, CleanWebDriverHelper, ProxySetting}

//TODO: switch to DriverPool! Tor cannot handle too many connection request.
sealed abstract class DriverFactory extends Serializable{

  final def newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {
    val result = _newInstance(capabilities, spooky)

    result
  }

  def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver
}

object DriverFactories {

  object PhantomJS {

    def phantomJSPath(fileName: String) = Option(System.getenv("PHANTOMJS_PATH"))
      .getOrElse{
      LoggerFactory.getLogger(this.getClass).info("$PHANTOMJS_PATH does not exist, using tempfile instead")
      SparkFiles.get(fileName)
    }

    //used in sc.addFile(...)
    @transient val phantomJSUrl = Option(System.getenv("PHANTOMJS_PATH")).getOrElse {
      sys.error("Please set PHANTOMJS_PATH on Spark driver system environment")
    }
    //TODO: download it from public resource
    @transient val phantomJSFileName = new Path(phantomJSUrl).getName
  }

  case class PhantomJS(
                        fileName: String = PhantomJS.phantomJSFileName,
                        loadImages: Boolean = false
                        )
    extends DriverFactory {

    def exePath = PhantomJS.phantomJSPath(fileName)

    @transient lazy val baseCaps = new DesiredCapabilities(BrowserType.PHANTOMJS, "", Platform.ANY)
    baseCaps.setJavascriptEnabled(true); //< not really needed: JS enabled by default
    baseCaps.setCapability(CapabilityType.SUPPORTS_FINDING_BY_CSS, true)
    //  baseCaps.setCapability(CapabilityType.HAS_NATIVE_EVENTS, false)
    baseCaps.setCapability(TAKES_SCREENSHOT, true)
    baseCaps.setCapability(ACCEPT_SSL_CERTS, true)
    baseCaps.setCapability(SUPPORTS_ALERTS, true)
    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY, exePath)
    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "loadImages", loadImages)

    //    baseCaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX+"resourceTimeout", Const.resourceTimeout*1000)

    def newCap(capabilities: Capabilities, spooky: SpookyContext): DesiredCapabilities = {
      val result = new DesiredCapabilities(baseCaps)

      result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX+"resourceTimeout", spooky.conf.remoteResourceTimeout*1000)

      val userAgent = spooky.conf.userAgent
      if (userAgent != null) result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "userAgent", userAgent)

      val proxy = spooky.conf.proxy()

      if (proxy != null)
        result.setCapability(
          PhantomJSDriverService.PHANTOMJS_CLI_ARGS,
          Array("--proxy=" + proxy.addr+":"+proxy.port, "--proxy-type=" + proxy.protocol)
        )

      result.merge(capabilities)
    }

    //called from executors
    override def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {

      new PhantomJSDriver(newCap(capabilities, spooky)) with CleanWebDriverHelper
    }
  }

  case class HtmlUnit(
                       browser: BrowserVersion = BrowserVersion.FIREFOX_24
                       ) extends DriverFactory {

    val baseCaps = new DesiredCapabilities(BrowserType.HTMLUNIT, "", Platform.ANY)

    def newCap(capabilities: Capabilities, spooky: SpookyContext): DesiredCapabilities = {
      val result = new DesiredCapabilities(baseCaps)

      val userAgent = spooky.conf.userAgent
      //TODO: this is useless, need custom BrowserVersion
      //see http://stackoverflow.com/questions/12853715/setting-user-agent-for-htmlunitdriver-selenium
      if (userAgent != null) result.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "userAgent", userAgent)

      val proxy: ProxySetting = spooky.conf.proxy()

      if (proxy != null) {
        result.setCapability(PROXY, proxy.toSeleniumProxy)
      }

      result.merge(capabilities)
    }

    override def _newInstance(capabilities: Capabilities, spooky: SpookyContext): CleanWebDriver = {

      val cap = newCap(capabilities, spooky)
      val driver = new HtmlUnitDriver(browser) with CleanWebDriverHelper
      driver.setJavascriptEnabled(true)
      driver.setProxySettings(Proxy.extractFrom(cap))

      driver
    }
  }

  ////just for debugging
  ////a bug in this driver has caused it unusable in Firefox 32
  //object FirefoxDriverFactory extends DriverFactory {
  //
  //  val baseCaps = new DesiredCapabilities
  //  //  baseCaps.setJavascriptEnabled(true);                //< not really needed: JS enabled by default
  //  //  baseCaps.setCapability(CapabilityType.SUPPORTS_FINDING_BY_CSS,true)
  //
  //  //  val FirefoxRootPath = "/usr/lib/phantomjs/"
  //  //  baseCaps.setCapability("webdriver.firefox.bin", "firefox");
  //  //  baseCaps.setCapability("webdriver.firefox.profile", "WebDriver");
  //
  //  override def newInstance(capabilities: Capabilities, spooky: SpookyContext): WebDriver = {
  //    val newCap = baseCaps.merge(capabilities)
  //
  //    Utils.retry(Const.DFSInPartitionRetry) {
  //      Utils.withDeadline(spooky.distributedResourceTimeout) {new FirefoxDriver(newCap)}
  //    }
  //  }
  //}

}
{'fraction_non_alphanumeric': 0.09086651053864168, 'fraction_numerical': 0.005152224824355972, 'mean_word_length': 3.8729554963864588, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1891313', 'n_tokens_mistral': 4053, 'n_tokens_neox': 3869, 'n_words': 820}