<commit_before>package org.tribbloid.spookystuff.integration

import java.net.URLEncoder

import org.tribbloid.spookystuff.SpookyContext
import org.tribbloid.spookystuff.actions._
import org.tribbloid.spookystuff.dsl._

import scala.concurrent.duration

/**
 * Created by peng on 12/14/14.
 */
class FetchInteractionsIT extends IntegrationSuite{

  override def doMain(spooky: SpookyContext): Unit = {

    val RDD = spooky
      .fetch(
        Visit("http://www.wikipedia.org")
          +> TextInput("input#searchInput","深度学习")
          +> DropDownSelect("select#searchLanguage","zh")
          +> Submit("input.formBtn")
      ).persist()

    val pageRows = RDD.collect()

    val finishTime = System.currentTimeMillis()
    assert(pageRows.length === 1)
    assert(pageRows(0).pages.length === 1)
    val uri = pageRows(0).pages(0).uri
    assert((uri === "http://zh.wikipedia.org/wiki/深度学习") || uri === "http://zh.wikipedia.org/wiki/"+URLEncoder.encode("深度学习", "UTF-8"))
    assert(pageRows(0).pages(0).name === "Snapshot()")
    val pageTime = pageRows(0).pages.head.timestamp.getTime
    assert(pageTime < finishTime)
    assert(pageTime > finishTime-120000) //long enough even after the second time it is retrieved from s3 cache

    Thread.sleep(10000) //this delay is necessary to circumvent eventual consistency of HDFS-based cache

    val RDDAppended = RDD
      .fetch(
        Visit("http://www.wikipedia.org")
          +> TextInput("input#searchInput","深度学习")
          +> DropDownSelect("select#searchLanguage","zh")
          +> Submit("input.formBtn")
          +> Snapshot().as('b),
        joinType = Append
      )

    val appendedRows = RDDAppended.collect()

    assert(appendedRows.length === 2)
    assert(appendedRows(0).pages(0).copy(timestamp = null, content = null, saved = null)<commit_msg>fix a problem in integration test caused by a link to https<commit_after>package org.tribbloid.spookystuff.integration

import java.net.URLEncoder

import org.tribbloid.spookystuff.SpookyContext
import org.tribbloid.spookystuff.actions._
import org.tribbloid.spookystuff.dsl._

import scala.concurrent.duration

/**
 * Created by peng on 12/14/14.
 */
class FetchInteractionsIT extends IntegrationSuite{

  override def doMain(spooky: SpookyContext): Unit = {

    val RDD = spooky
      .fetch(
        Visit("http://www.wikipedia.org")
          +> TextInput("input#searchInput","深度学习")
          +> DropDownSelect("select#searchLanguage","zh")
          +> Submit("input.formBtn")
      ).persist()

    val pageRows = RDD.collect()

    val finishTime = System.currentTimeMillis()
    assert(pageRows.length === 1)
    assert(pageRows(0).pages.length === 1)
    val uri = pageRows(0).pages(0).uri
    assert((uri endsWith "zh.wikipedia.org/wiki/深度学习") || (uri endsWith "zh.wikipedia.org/wiki/"+URLEncoder.encode("深度学习", "UTF-8")))
    assert(pageRows(0).pages(0).name === "Snapshot()")
    val pageTime = pageRows(0).pages.head.timestamp.getTime
    assert(pageTime < finishTime)
    assert(pageTime > finishTime-120000) //long enough even after the second time it is retrieved from s3 cache

    Thread.sleep(10000) //this delay is necessary to circumvent eventual consistency of HDFS-based cache

    val RDDAppended = RDD
      .fetch(
        Visit("http://www.wikipedia.org")
          +> TextInput("input#searchInput","深度学习")
          +> DropDownSelect("select#searchLanguage","zh")
          +> Submit("input.formBtn")
          +> Snapshot().as('b),
        joinType = Append
      )

    val appendedRows = RDDAppended.collect()

    assert(appendedRows.length === 2)
    assert(appendedRows(0).pages(0).copy(timestamp = null, content = null, saved = null)
{'fraction_non_alphanumeric': 0.13852813852813853, 'fraction_numerical': 0.016233766233766232, 'mean_word_length': 3.422248803827751, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6965665', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1186, 'n_words': 230}