<commit_before>package org.broadinstitute.clio.integrationtest

import java.io.File
import java.nio.file.attribute.PosixFilePermissions
import java.nio.file.{Files, Path, Paths, StandardOpenOption}
import java.time.OffsetDateTime

import akka.http.scaladsl.model.Uri
import com.dimafeng.testcontainers.DockerComposeContainer
import io.circe.Json
import org.broadinstitute.clio.client.util.IoUtil
import org.broadinstitute.clio.server.dataaccess.elasticsearch.{
  ClioDocument,
  ElasticsearchIndex
}
import org.broadinstitute.clio.util.json.ModelAutoDerivation
import org.broadinstitute.clio.util.model.UpsertId
import org.junit.runner.Description
import org.testcontainers.utility.Base58

import scala.collection.JavaConverters._

/**
  * Extension of [[com.dimafeng.testcontainers.DockerComposeContainer]],
  * with settings tailored for our integration tests, since
  * testcontainers-scala doesn't make the settings configurable.
  */
class ClioDockerComposeContainer(
  composeFile: File,
  elasticsearchHostname: String,
  exposedServices: Map[String, Int],
  seededDocuments: Map[ElasticsearchIndex[_], Seq[Json]] = Map.empty
) extends DockerComposeContainer(
      Seq(composeFile),
      exposedServices,
      Base58.randomString(6).toLowerCase()
    )
    with ModelAutoDerivation {

  /** Log files to mount into the Elasticsearch containers. */
  private val esLogs = for {
    service <- Seq("elasticsearch1", "elasticsearch2")
    filename <- Seq("docker-cluster.log", "docker-cluster_access.log")
  } yield {
    Paths.get(ClioBuildInfo.logDir, service, filename)
  }

  /*
   * Skip "docker pull" because it'll fail if we're testing against a non-
   * pushed version of clio-server.
   */
  container.withPull(false)

  /*
   * Testcontainers doesn't pass env vars onto the docker-compose container it
   * spins up, and we need the vars in order to fill in our compose config.
   * Using the local install gets the vars to pass through.
   */
  container.withLocalCompose(true)

  /*
   * Most of the environment variables needed to fill in our compose files are
   * constants across test suites and passed in by SBT (because they deal with
   * versions / file paths), but the elasticsearch host to test against might vary
   * from suite to suite so it comes in as a constructor parameter instead.
   */
  container.withEnv(
    Map(
      ClioDockerComposeContainer.clioVersionVariable -> ClioBuildInfo.version,
      ClioDockerComposeContainer.elasticsearchVersionVariable -> ClioBuildInfo.elasticsearchVersion,
      ClioDockerComposeContainer.elasticsearchHostVariable -> elasticsearchHostname,
      ClioDockerComposeContainer.configDirVariable -> ClioBuildInfo.confDir,
      ClioDockerComposeContainer.logDirVariable -> ClioBuildInfo.logDir,
      ClioDockerComposeContainer.clioLogFileVariable -> ClioDockerComposeContainer.clioLog.toString,
      ClioDockerComposeContainer.persistenceDirVariable -> ClioBuildInfo.persistenceDir
    ).asJava
  )

  /**
    * Make sure log files exist with global rw permissions before starting containers.
    *
    * Global rw permissions are needed so the integration tests can run on Jenkins,
    * which uses an older version of Docker that bind-mounts files into containers
    * with weird uids and gids. If permissions aren't set properly, log4j will fail
    * to initialize in the ES containers and tests will fail to start.
    */
  override def starting()(implicit description: Description): Unit = {
    val rootPersistenceDir = Paths.get(ClioBuildInfo.persistenceDir)

    Seq(Paths.get(ClioBuildInfo.logDir), rootPersistenceDir).foreach { dir =>
      if (Files.exists(dir)) {
        IoUtil.deleteDirectoryRecursively(dir)
      }
      val _ = Files.createDirectories(dir)
    }

    /*
     * Simulate spreading pre-seeded documents over time.
     *
     * NOTE: The spread calculation is meant to work around a problem with case-sensitivity
     * when running this test on OS X. Our `UpsertId`s are case-sensitive, but HFS / APFS are
     * case-insensitive by default. This can cause naming collisions when generating a ton of
     * IDs at once (like in these tests). By spreading documents into bins of 26, we try to
     * avoid the possibility of two IDs differing only in the case of the last byte being dropped
     * into the same day-directory.
     */
    val daySpread = (seededDocuments.map(_._2.size).max / 26).toLong
    val today: OffsetDateTime = OffsetDateTime.now()
    val earliest: OffsetDateTime = today.minusDays(daySpread)

    seededDocuments.foreach {
      case (index, documents) =>
        val documentCount = documents.length

        documents.zipWithIndex.foreach {
          case (json, i) => {
            val dateDir = index.persistenceDirForDatetime(
              earliest.plusDays(i.toLong / (documentCount.toLong / daySpread))
            )

            val writeDir =
              Files.createDirectories(rootPersistenceDir.resolve(s"$dateDir/"))
            val upsertId = json.hcursor
              .get[UpsertId](ClioDocument.UpsertIdElasticSearchName)
              .fold(throw _, identity)

            val _ = Files.write(
              writeDir.resolve(ClioDocument.persistenceFilename(upsertId)),
              defaultPrinter.pretty(json).getBytes,
              StandardOpenOption.CREATE_NEW,
              StandardOpenOption.WRITE
            )
          }
        }
    }

    val logPaths = ClioDockerComposeContainer.clioLog +: esLogs
    logPaths.foreach { log =>
      if (!Files.exists(log.getParent)) {
        val _ = Files.createDirectories(log.getParent)
      }
      if (!Files.exists(log)) {
        val _ = Files.createFile(log)
        Files.setPosixFilePermissions(
          log,
          PosixFilePermissions.fromString("rw-rw-rw-")
        )
      }
    }
    super.starting()
  }

  /**
    * After the container stops, move its logs and storage so
    * they don't interfere with subsequent test suites.
    */
  override def finished()(implicit description: Description): Unit = {
    super.finished()
    Seq(ClioBuildInfo.logDir, ClioBuildInfo.persistenceDir).foreach { dirPrefix =>
      val target = Paths.get(s"$dirPrefix-${description.getDisplayName}")
      if (Files.exists(target)) {
        IoUtil.deleteDirectoryRecursively(target)
      }
      val _ = Files.move(Paths.get(dirPrefix), target)
    }<commit_msg>Fix non-recovery Docker integration tests. (#215)<commit_after>package org.broadinstitute.clio.integrationtest

import java.io.File
import java.nio.file.attribute.PosixFilePermissions
import java.nio.file.{Files, Path, Paths, StandardOpenOption}
import java.time.OffsetDateTime

import akka.http.scaladsl.model.Uri
import com.dimafeng.testcontainers.DockerComposeContainer
import io.circe.Json
import org.broadinstitute.clio.client.util.IoUtil
import org.broadinstitute.clio.server.dataaccess.elasticsearch.{
  ClioDocument,
  ElasticsearchIndex
}
import org.broadinstitute.clio.util.json.ModelAutoDerivation
import org.broadinstitute.clio.util.model.UpsertId
import org.junit.runner.Description
import org.testcontainers.utility.Base58

import scala.collection.JavaConverters._

/**
  * Extension of [[com.dimafeng.testcontainers.DockerComposeContainer]],
  * with settings tailored for our integration tests, since
  * testcontainers-scala doesn't make the settings configurable.
  */
class ClioDockerComposeContainer(
  composeFile: File,
  elasticsearchHostname: String,
  exposedServices: Map[String, Int],
  seededDocuments: Map[ElasticsearchIndex[_], Seq[Json]] = Map.empty
) extends DockerComposeContainer(
      Seq(composeFile),
      exposedServices,
      Base58.randomString(6).toLowerCase()
    )
    with ModelAutoDerivation {

  /** Log files to mount into the Elasticsearch containers. */
  private val esLogs = for {
    service <- Seq("elasticsearch1", "elasticsearch2")
    filename <- Seq("docker-cluster.log", "docker-cluster_access.log")
  } yield {
    Paths.get(ClioBuildInfo.logDir, service, filename)
  }

  /*
   * Skip "docker pull" because it'll fail if we're testing against a non-
   * pushed version of clio-server.
   */
  container.withPull(false)

  /*
   * Testcontainers doesn't pass env vars onto the docker-compose container it
   * spins up, and we need the vars in order to fill in our compose config.
   * Using the local install gets the vars to pass through.
   */
  container.withLocalCompose(true)

  /*
   * Most of the environment variables needed to fill in our compose files are
   * constants across test suites and passed in by SBT (because they deal with
   * versions / file paths), but the elasticsearch host to test against might vary
   * from suite to suite so it comes in as a constructor parameter instead.
   */
  container.withEnv(
    Map(
      ClioDockerComposeContainer.clioVersionVariable -> ClioBuildInfo.version,
      ClioDockerComposeContainer.elasticsearchVersionVariable -> ClioBuildInfo.elasticsearchVersion,
      ClioDockerComposeContainer.elasticsearchHostVariable -> elasticsearchHostname,
      ClioDockerComposeContainer.configDirVariable -> ClioBuildInfo.confDir,
      ClioDockerComposeContainer.logDirVariable -> ClioBuildInfo.logDir,
      ClioDockerComposeContainer.clioLogFileVariable -> ClioDockerComposeContainer.clioLog.toString,
      ClioDockerComposeContainer.persistenceDirVariable -> ClioBuildInfo.persistenceDir
    ).asJava
  )

  /**
    * Make sure log files exist with global rw permissions before starting containers.
    *
    * Global rw permissions are needed so the integration tests can run on Jenkins,
    * which uses an older version of Docker that bind-mounts files into containers
    * with weird uids and gids. If permissions aren't set properly, log4j will fail
    * to initialize in the ES containers and tests will fail to start.
    */
  override def starting()(implicit description: Description): Unit = {
    val rootPersistenceDir = Paths.get(ClioBuildInfo.persistenceDir)

    Seq(Paths.get(ClioBuildInfo.logDir), rootPersistenceDir).foreach { dir =>
      if (Files.exists(dir)) {
        IoUtil.deleteDirectoryRecursively(dir)
      }
      val _ = Files.createDirectories(dir)
    }

    if (seededDocuments.nonEmpty) {
      /*
       * Simulate spreading pre-seeded documents over time.
       *
       * NOTE: The spread calculation is meant to work around a problem with case-sensitivity
       * when running this test on OS X. Our `UpsertId`s are case-sensitive, but HFS / APFS are
       * case-insensitive by default. This can cause naming collisions when generating a ton of
       * IDs at once (like in these tests). By spreading documents into bins of 26, we try to
       * avoid the possibility of two IDs differing only in the case of the last byte being dropped
       * into the same day-directory.
       */
      val daySpread = (seededDocuments.map(_._2.size).max / 26).toLong
      val today: OffsetDateTime = OffsetDateTime.now()
      val earliest: OffsetDateTime = today.minusDays(daySpread)

      seededDocuments.foreach {
        case (index, documents) =>
          val documentCount = documents.length

          documents.zipWithIndex.foreach {
            case (json, i) => {
              val dateDir = index.persistenceDirForDatetime(
                earliest.plusDays(i.toLong / (documentCount.toLong / daySpread))
              )

              val writeDir =
                Files.createDirectories(rootPersistenceDir.resolve(s"$dateDir/"))
              val upsertId = json.hcursor
                .get[UpsertId](ClioDocument.UpsertIdElasticSearchName)
                .fold(throw _, identity)

              val _ = Files.write(
                writeDir.resolve(ClioDocument.persistenceFilename(upsertId)),
                defaultPrinter.pretty(json).getBytes,
                StandardOpenOption.CREATE_NEW,
                StandardOpenOption.WRITE
              )
            }
          }
      }
    }

    val logPaths = ClioDockerComposeContainer.clioLog +: esLogs
    logPaths.foreach { log =>
      if (!Files.exists(log.getParent)) {
        val _ = Files.createDirectories(log.getParent)
      }
      if (!Files.exists(log)) {
        val _ = Files.createFile(log)
        Files.setPosixFilePermissions(
          log,
          PosixFilePermissions.fromString("rw-rw-rw-")
        )
      }
    }
    super.starting()
  }

  /**
    * After the container stops, move its logs and storage so
    * they don't interfere with subsequent test suites.
    */
  override def finished()(implicit description: Description): Unit = {
    super.finished()
    Seq(ClioBuildInfo.logDir, ClioBuildInfo.persistenceDir).foreach { dirPrefix =>
      val target = Paths.get(s"$dirPrefix-${description.getDisplayName}")
      if (Files.exists(target)) {
        IoUtil.deleteDirectoryRecursively(target)
      }
      val _ = Files.move(Paths.get(dirPrefix), target)
    }
{'fraction_non_alphanumeric': 0.08396591789310612, 'fraction_numerical': 0.0022463206816421377, 'mean_word_length': 3.1501125040180007, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1379651', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3600, 'n_words': 961}