<commit_before>package controllers

import java.io.ByteArrayInputStream
import scala.collection.JavaConversions.iterableAsScalaIterable
import org.specs2.mutable.Specification
import javax.jcr.Session
import models.User
import scala.concurrent.Await
import scala.concurrent.Promise
import scala.concurrent.duration.Duration
import play.api.mvc.AnyContentAsEmpty
import play.api.mvc.AsyncResult
import play.api.mvc.ChunkedResult
import play.api.mvc.Results
import play.api.mvc.SimpleResult
import play.api.mvc.Results.EmptyContent
import play.api.test.FakeHeaders
import play.api.test.FakeRequest
import play.api.test.Helpers.GET
import play.api.test.Helpers.OK
import play.api.test.Helpers.charset
import play.api.test.Helpers.contentAsString
import play.api.test.Helpers.contentType
import play.api.test.Helpers.header
import play.api.test.Helpers.route
import play.api.test.Helpers.status
import play.api.test.Helpers.writeableOf_AnyContentAsEmpty
import test.AorraScalaHelper.FakeAorraApp
import test.AorraScalaHelper.asAdminUser
import test.AorraScalaHelper.filestore
import java.util.concurrent.TimeoutException
import java.util.concurrent.TimeUnit
import play.api.libs.iteratee.{Enumeratee, Iteratee}
import scala.concurrent.Future
import java.util.zip.{ZipInputStream, ZipEntry}
import helpers.FileStoreHelper.{XLS_MIME_TYPE, XLSX_MIME_TYPE}
import java.io.FileInputStream
import scala.collection.immutable.StringOps
import akka.util.Timeout

class ArchiveAsyncSpec extends Specification {

  implicit val timeout = Timeout(30000)

  "chart archive download" should {

    "require login" in new FakeAorraApp {
      asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        val folder = filestore.getManager(session).getRoot()

        val Some(result) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString))

        status(result) must equalTo(303)
        header("Location", result) must beSome("/login")
      }
    }

    "return archive" in new FakeAorraApp {
      asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        import scala.concurrent.ExecutionContext.Implicits.global

        val folder = filestore.getManager(session).getRoot()
        folder.createFile("marine.xlsx", XLSX_MIME_TYPE,
            new FileInputStream("test/marine.xlsx"))

        val Some(result: Future[SimpleResult]) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString,
            rh, AnyContentAsEmpty))

        status(result) must equalTo(OK);
        contentType(result) must beSome("application/zip")
        header("Cache-Control", result) must
          beSome("max-age=0, must-revalidate");

        val chunkedResult = Await.result(result, Duration(30, TimeUnit.SECONDS))

        var zipBytes: Array[Byte] = Array.emptyByteArray
        val byteCollector = Iteratee.fold[Array[Byte], Unit](zipBytes) {
          (_, chunkedBytes) => zipBytes = zipBytes ++ chunkedBytes
        }

        val futureResult = chunkedResult.body.through(Results.dechunk)
            .run(byteCollector)

        Await.result(futureResult, Duration(30, TimeUnit.SECONDS))

        zipBytes.slice(0, 2) must equalTo("PK".getBytes)

        val filepaths: Set[String] = {
          val zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))
          Stream.continually(zis.getNextEntry)
            .takeWhile(_ != null)
            .map(_.getName)
            .toSet
        }

        val filenamesByExt = filepaths.map { filepath =>
          filepath.split("/") match {
            case Array(folderId, filename) =>
              folderId must equalTo(folder.getIdentifier())
              filename
          }
        }.groupBy { filename =>
          filename.split("\\.").last
        }

        filenamesByExt must haveKeys("csv", "docx", "emf", "html", "png", "svg")

        filenamesByExt.forall { case (k, v) =>
          v must haveSize(7)
        }
      }
    }

    "handle charts with extra parameters" in new FakeAorraApp {
        asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        val folder = filestore.getManager(session).getRoot()
        folder.createFile("seagrass_cover.xls", XLS_MIME_TYPE,
            new FileInputStream("test/seagrass_cover.xls"))

        val Some(result) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString,
            rh, AnyContentAsEmpty))

        status(result) must equalTo(OK);
        contentType(result) must beSome("application/zip")
        header("Cache-Control", result) must
          beSome("max-age=0, must-revalidate");

        result match {
          case AsyncResult(_) => // all good
          case _ => failure("Should have an asynchronous result.")
        }

        val chunkedResult = await(result.asInstanceOf[AsyncResult].result,
                30, TimeUnit.SECONDS).asInstanceOf[ChunkedResult[Array[Byte]]]

        var zipBytes: Array[Byte] = Array.emptyByteArray
        val byteCollector = Iteratee.fold[Array[Byte], Unit](zipBytes) {
          (_, chunkedBytes) => zipBytes = zipBytes ++ chunkedBytes
        }

        val promisedIteratee = chunkedResult.chunks(byteCollector)
            .asInstanceOf[Promise[Iteratee[Array[Byte], Unit]]]

        await(promisedIteratee.future, 30, TimeUnit.SECONDS)

        zipBytes.slice(0, 2) must equalTo("PK".getBytes)

        val filepaths: Set[String] = {
          val zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))
          Stream.continually(zis.getNextEntry)
            .takeWhile(_ != null)
            .map(_.getName)
            .toSet
        }

        val filenamesByExt = filepaths.map { filepath =>
          filepath.split("/") match {
            case Array(folderId, filename) =>
              folderId must equalTo(folder.getIdentifier())
              filename
          }
        }.groupBy { filename =>
          filename.split("\\.").last
        }

        filenamesByExt must haveKeys("docx", "emf", "png", "svg")

        filenamesByExt.forall { case (k, v) =>
          v must haveSize(15)
        }
      }

    }

  }

}<commit_msg>Cleaning up and documenting how rewritten async test works.<commit_after>package controllers

import java.io.ByteArrayInputStream
import scala.collection.JavaConversions.iterableAsScalaIterable
import org.specs2.mutable.Specification
import javax.jcr.Session
import models.User
import scala.concurrent.Await
import scala.concurrent.Promise
import scala.concurrent.duration.Duration
import play.api.mvc.AnyContentAsEmpty
import play.api.mvc.AsyncResult
import play.api.mvc.ChunkedResult
import play.api.mvc.Results
import play.api.mvc.SimpleResult
import play.api.mvc.Results.EmptyContent
import play.api.test.FakeHeaders
import play.api.test.FakeRequest
import play.api.test.Helpers.GET
import play.api.test.Helpers.OK
import play.api.test.Helpers.charset
import play.api.test.Helpers.contentAsString
import play.api.test.Helpers.contentType
import play.api.test.Helpers.header
import play.api.test.Helpers.route
import play.api.test.Helpers.status
import play.api.test.Helpers.writeableOf_AnyContentAsEmpty
import test.AorraScalaHelper.FakeAorraApp
import test.AorraScalaHelper.asAdminUser
import test.AorraScalaHelper.filestore
import java.util.concurrent.TimeoutException
import java.util.concurrent.TimeUnit
import play.api.libs.iteratee.{Enumeratee, Iteratee}
import scala.concurrent.Future
import java.util.zip.{ZipInputStream, ZipEntry}
import helpers.FileStoreHelper.{XLS_MIME_TYPE, XLSX_MIME_TYPE}
import java.io.FileInputStream
import scala.collection.immutable.StringOps
import akka.util.Timeout

class ArchiveAsyncSpec extends Specification {

  implicit val timeout = Timeout(30000)

  "chart archive download" should {

    "require login" in new FakeAorraApp {
      asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        val folder = filestore.getManager(session).getRoot()

        val Some(result) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString))

        status(result) must equalTo(303)
        header("Location", result) must beSome("/login")
      }
    }

    "return archive" in new FakeAorraApp {
      asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        import scala.concurrent.ExecutionContext.Implicits.global

        val folder = filestore.getManager(session).getRoot()
        folder.createFile("marine.xlsx", XLSX_MIME_TYPE,
            new FileInputStream("test/marine.xlsx"))

        val Some(futureResult: Future[SimpleResult]) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString,
            rh, AnyContentAsEmpty))

        status(futureResult) must equalTo(OK);
        contentType(futureResult) must beSome("application/zip")
        header("Cache-Control", futureResult) must
          beSome("max-age=0, must-revalidate");

        // Need to work with actual result
        val result = Await.result(futureResult, Duration(30, TimeUnit.SECONDS))

        // Take "Transfer-Encoding: chunked" bytes, dechunk and collect
        val futureZipBytes = result.body.through(Results.dechunk)
            .run(Iteratee.consume[Array[Byte]]())

        // Wait for 30 seconds for all bytes to be collected
        val zipBytes: Array[Byte] =
          Await.result(futureZipBytes, Duration(30, TimeUnit.SECONDS))

        // Check magic number of sent file
        zipBytes.slice(0, 2) must equalTo("PK".getBytes)

        val filepaths: Set[String] = {
          val zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))
          Stream.continually(zis.getNextEntry)
            .takeWhile(_ != null)
            .map(_.getName)
            .toSet
        }

        val filenamesByExt = filepaths.map { filepath =>
          filepath.split("/") match {
            case Array(folderId, filename) =>
              folderId must equalTo(folder.getIdentifier())
              filename
          }
        }.groupBy { filename =>
          filename.split("\\.").last
        }

        filenamesByExt must haveKeys("csv", "docx", "emf", "html", "png", "svg")

        filenamesByExt.forall { case (k, v) =>
          v must haveSize(7)
        }
      }
    }

    "handle charts with extra parameters" in new FakeAorraApp {
        asAdminUser { (session: Session, user: User, rh: FakeHeaders) =>
        val folder = filestore.getManager(session).getRoot()
        folder.createFile("seagrass_cover.xls", XLS_MIME_TYPE,
            new FileInputStream("test/seagrass_cover.xls"))

        val Some(result) = route(FakeRequest(GET,
            routes.ArchiveAsync.chartArchive(folder.getIdentifier).toString,
            rh, AnyContentAsEmpty))

        status(result) must equalTo(OK);
        contentType(result) must beSome("application/zip")
        header("Cache-Control", result) must
          beSome("max-age=0, must-revalidate");

        result match {
          case AsyncResult(_) => // all good
          case _ => failure("Should have an asynchronous result.")
        }

        val chunkedResult = await(result.asInstanceOf[AsyncResult].result,
                30, TimeUnit.SECONDS).asInstanceOf[ChunkedResult[Array[Byte]]]

        var zipBytes: Array[Byte] = Array.emptyByteArray
        val byteCollector = Iteratee.fold[Array[Byte], Unit](zipBytes) {
          (_, chunkedBytes) => zipBytes = zipBytes ++ chunkedBytes
        }

        val promisedIteratee = chunkedResult.chunks(byteCollector)
            .asInstanceOf[Promise[Iteratee[Array[Byte], Unit]]]

        await(promisedIteratee.future, 30, TimeUnit.SECONDS)

        zipBytes.slice(0, 2) must equalTo("PK".getBytes)

        val filepaths: Set[String] = {
          val zis = new ZipInputStream(new ByteArrayInputStream(zipBytes))
          Stream.continually(zis.getNextEntry)
            .takeWhile(_ != null)
            .map(_.getName)
            .toSet
        }

        val filenamesByExt = filepaths.map { filepath =>
          filepath.split("/") match {
            case Array(folderId, filename) =>
              folderId must equalTo(folder.getIdentifier())
              filename
          }
        }.groupBy { filename =>
          filename.split("\\.").last
        }

        filenamesByExt must haveKeys("docx", "emf", "png", "svg")

        filenamesByExt.forall { case (k, v) =>
          v must haveSize(15)
        }
      }

    }

  }

}
{'fraction_non_alphanumeric': 0.10658682634730539, 'fraction_numerical': 0.004311377245508982, 'mean_word_length': 2.873222016079159, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7473368', 'n_tokens_mistral': 3748, 'n_tokens_neox': 3638, 'n_words': 728}