<commit_before>/**
 * Copyright (C) 2010-2011 LShift Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.lshift.diffa.agent.itest

import org.junit.Assert._
import net.lshift.diffa.agent.itest.support.TestConstants._
import java.lang.String
import support.TestEnvironment
import javax.mail.Session
import java.io.{File, FileInputStream}
import javax.mail.internet.MimeMessage
import util.matching.Regex
import org.slf4j.{Logger, LoggerFactory}
import java.net.URI
import org.junit.{Before, Test}
import net.lshift.diffa.kernel.participants.ParticipantType
import net.lshift.diffa.agent.client.DifferencesRestClient
import net.lshift.diffa.kernel.differencing.{ZoomCache, PairScanState, DifferenceEvent}
import net.lshift.diffa.kernel.differencing.ZoomCache._
import scala.collection.JavaConversions._
import java.util.{UUID, Properties}
import org.joda.time.{DateTimeZone, DateTime}

/**
 * Tests that can be applied to an environment to validate that differencing functionality works appropriately.
 */
trait CommonDifferenceTests {

  val log:Logger = LoggerFactory.getLogger(getClass)

  val mailDir = System.getProperty("diffa.maildir")
  log.debug("Using maildir: " + mailDir)
  
  val messageDir = (new File(mailDir)).getAbsoluteFile

  /**
   * The environment under test.
   */
  def env:TestEnvironment

  @Before
  def deleteMails() = {
    if (messageDir.exists) {
      messageDir.listFiles.foreach(f => f.delete)
    }
  }

  def getReport(from:DateTime, until:DateTime) : Array[DifferenceEvent]= {
    runScanAndWaitForCompletion(yearAgo, today)
    env.diffClient.getEvents(env.pairKey, from, until, 0, 100)
  }

  @Test
  def shouldFindNoDifferencesInEmptyParticipants {
    val diffs = getReport(yearAgo, today)
    assertNotNull(diffs)
    assertTrue(diffs.isEmpty)
  }

  @Test
  def detectionTimeShouldBeMatchTheMostRecentUpdatedTimeOnAParticipatingEntity = {
    val diffs = getVerifiedDiffs()
    assertNotNull(diffs(0))
    val detectionTime = diffs(0).detectedAt
    // TODO On a rainy day, look into why using a millisecond comparison is necessary
    assertEquals(yesterday.getMillis, detectionTime.getMillis)
  }

  @Test
  def shouldFindDifferencesInDifferingParticipants {
    val diffs = getVerifiedDiffs()
    val seqId = diffs(0).seqId

    val detail = env.diffClient.eventDetail(seqId, ParticipantType.UPSTREAM)
    assertNotNull(diffs)

    assertEquals("abcdef", detail)

    val fileList = messageDir.listFiles
    assertNotNull("File list was null for dir: " + messageDir, fileList)
    // #338 This notification test is broken since the default quiet time was set to a non-zero value,
    // but unfortuneately, this requires the config to be dynamically updateable.
    //assertEquals(1, fileList.size)
    //testForLink(fileList(0))
  }

  @Test
  def differencesShouldTile = {
    val diffs = getVerifiedDiffs()
    assertNotNull(diffs)

    val zoomLevel = ZoomCache.QUARTER_HOURLY
    val tiles = env.diffClient.getZoomedTiles(yesterday, tomorrow, zoomLevel)
    assertNotNull(tiles)
    assertNotNull(tiles(env.pairKey))
  }

  @Test
  def differencesShouldTileAtEachLevel = {

    val rightNow = new DateTime(2011,7,8,15,0,0,0, DateTimeZone.UTC)

    val events = 480 // 8 hours * 60 minutes, i.e. will get split at 8-hourly level, but fits into a daily column
    val columns = 32

    val expectedZoomedViews = Map(
      DAILY        -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480), // 24 hour interval
      EIGHT_HOURLY -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 420), // 0-8h, 8-16h intervals
      FOUR_HOURLY -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 240, 180)  // 8-12h,12-16h
    )

    resetPair
    env.clearParticipants


    val upperBound = rightNow.plusMinutes(1)

    for (i <- 1 to events) {
      val timestamp = rightNow.minusMinutes(i)
      env.upstream.addEntity("id-" + i, yesterday, "ss", timestamp, "abcdef")
    }

    val fullLowerBound = rightNow.minusMinutes(ZoomCache.zoom(DAILY) * columns)
    runScanAndWaitForCompletion(fullLowerBound, upperBound, 100, 100)

    expectedZoomedViews.foreach{ case (zoomLevel, expectedBlobs) => {
      val lowerBound = upperBound.minusMinutes(ZoomCache.zoom(zoomLevel) * columns)
      val tiles = env.diffClient.getZoomedTiles(lowerBound, upperBound, zoomLevel)
      val retrievedBlobs = tiles(env.pairKey)
      assertEquals("Zoom level %s ".format(zoomLevel), expectedBlobs,retrievedBlobs)
    }}
  }

  private def resetPair = {
    try {<commit_msg>[#359] Added a few more data points<commit_after>/**
 * Copyright (C) 2010-2011 LShift Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.lshift.diffa.agent.itest

import org.junit.Assert._
import net.lshift.diffa.agent.itest.support.TestConstants._
import java.lang.String
import support.TestEnvironment
import javax.mail.Session
import java.io.{File, FileInputStream}
import javax.mail.internet.MimeMessage
import util.matching.Regex
import org.slf4j.{Logger, LoggerFactory}
import java.net.URI
import org.junit.{Before, Test}
import net.lshift.diffa.kernel.participants.ParticipantType
import net.lshift.diffa.agent.client.DifferencesRestClient
import net.lshift.diffa.kernel.differencing.{ZoomCache, PairScanState, DifferenceEvent}
import net.lshift.diffa.kernel.differencing.ZoomCache._
import scala.collection.JavaConversions._
import java.util.{UUID, Properties}
import org.joda.time.{DateTimeZone, DateTime}

/**
 * Tests that can be applied to an environment to validate that differencing functionality works appropriately.
 */
trait CommonDifferenceTests {

  val log:Logger = LoggerFactory.getLogger(getClass)

  val mailDir = System.getProperty("diffa.maildir")
  log.debug("Using maildir: " + mailDir)
  
  val messageDir = (new File(mailDir)).getAbsoluteFile

  /**
   * The environment under test.
   */
  def env:TestEnvironment

  @Before
  def deleteMails() = {
    if (messageDir.exists) {
      messageDir.listFiles.foreach(f => f.delete)
    }
  }

  def getReport(from:DateTime, until:DateTime) : Array[DifferenceEvent]= {
    runScanAndWaitForCompletion(yearAgo, today)
    env.diffClient.getEvents(env.pairKey, from, until, 0, 100)
  }

  @Test
  def shouldFindNoDifferencesInEmptyParticipants {
    val diffs = getReport(yearAgo, today)
    assertNotNull(diffs)
    assertTrue(diffs.isEmpty)
  }

  @Test
  def detectionTimeShouldBeMatchTheMostRecentUpdatedTimeOnAParticipatingEntity = {
    val diffs = getVerifiedDiffs()
    assertNotNull(diffs(0))
    val detectionTime = diffs(0).detectedAt
    // TODO On a rainy day, look into why using a millisecond comparison is necessary
    assertEquals(yesterday.getMillis, detectionTime.getMillis)
  }

  @Test
  def shouldFindDifferencesInDifferingParticipants {
    val diffs = getVerifiedDiffs()
    val seqId = diffs(0).seqId

    val detail = env.diffClient.eventDetail(seqId, ParticipantType.UPSTREAM)
    assertNotNull(diffs)

    assertEquals("abcdef", detail)

    val fileList = messageDir.listFiles
    assertNotNull("File list was null for dir: " + messageDir, fileList)
    // #338 This notification test is broken since the default quiet time was set to a non-zero value,
    // but unfortuneately, this requires the config to be dynamically updateable.
    //assertEquals(1, fileList.size)
    //testForLink(fileList(0))
  }

  @Test
  def differencesShouldTile = {
    val diffs = getVerifiedDiffs()
    assertNotNull(diffs)

    val zoomLevel = ZoomCache.QUARTER_HOURLY
    val tiles = env.diffClient.getZoomedTiles(yesterday, tomorrow, zoomLevel)
    assertNotNull(tiles)
    assertNotNull(tiles(env.pairKey))
  }

  @Test
  def differencesShouldTileAtEachLevel = {

    val rightNow = new DateTime(2011,7,8,15,0,0,0, DateTimeZone.UTC)
    val upperBound = rightNow.plusMinutes(1)

    val events = 480 // 8 hours * 60 minutes, i.e. will get split at 8-hourly level, but fits into a daily column
    val columns = 32

    val expectedZoomedViews = Map(
      DAILY          -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480), // 24 hour interval
      EIGHT_HOURLY   -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 420), // 0-8h, 8-16h intervals
      FOUR_HOURLY    -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 240, 180),  // 8-12h,12-16h
      TWO_HOURLY     -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 120, 120, 120, 60),  // 8-10h,10-12h,12-14h,14-16h
      HOURLY         -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 60, 60, 60, 60, 60, 60, 60, 0),  // every hour, remembering the extra minute
      HALF_HOURLY    -> List(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 0),  // every half hour
      QUARTER_HOURLY -> List(15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0)  // every quarter hour
    )

    resetPair
    env.clearParticipants

    for (i <- 1 to events) {
      val timestamp = rightNow.minusMinutes(i)
      env.upstream.addEntity("id-" + i, yesterday, "ss", timestamp, "abcdef")
    }

    val fullLowerBound = rightNow.minusMinutes(ZoomCache.zoom(DAILY) * columns)
    runScanAndWaitForCompletion(fullLowerBound, upperBound, 100, 100)

    expectedZoomedViews.foreach{ case (zoomLevel, expectedBlobs) => {
      val lowerBound = upperBound.minusMinutes(ZoomCache.zoom(zoomLevel) * columns)
      val tiles = env.diffClient.getZoomedTiles(lowerBound, upperBound, zoomLevel)
      val retrievedBlobs = tiles(env.pairKey)
      assertEquals("Zoom level %s ".format(zoomLevel), expectedBlobs,retrievedBlobs)
    }}
  }

  private def resetPair = {
    try {
{'fraction_non_alphanumeric': 0.11620988099531193, 'fraction_numerical': 0.05102776776054814, 'mean_word_length': 3.591473509933775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2772380', 'n_tokens_mistral': 4267, 'n_tokens_neox': 3687, 'n_words': 1129}