<commit_before>package uk.ac.wellcome.platform.snapshot_convertor.flow

import akka.actor.ActorSystem
import akka.stream.{ActorMaterializer, Materializer}
import akka.stream.scaladsl.{Sink, Source}
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.display.models.DisplayWork
import uk.ac.wellcome.models.{IdentifiedWork, IdentifierSchemes, SourceIdentifier, WorksIncludes}
import uk.ac.wellcome.test.fixtures.AkkaFixtures
import uk.ac.wellcome.test.utils.ExtendedPatience
import uk.ac.wellcome.utils.JsonUtil._

import scala.concurrent.{ExecutionContextExecutor, Future}

class ElasticsearchHitToDisplayWorkFlowTest
    extends FunSpec
    with Matchers
    with AkkaFixtures
    with ScalaFutures
    with ExtendedPatience {

  val includes = WorksIncludes(
    identifiers = true,
    thumbnail = true,
    items = true
  )

  it("creates a DisplayWork from a single hit") {
    withActorSystem { actorSystem =>
      implicit val executionContext: ExecutionContextExecutor =
        actorSystem.dispatcher
      implicit val system: ActorSystem = actorSystem
      implicit val materializer: Materializer =
        ActorMaterializer()(actorSystem)

      val flow = ElasticsearchHitToDisplayWorkFlow()

      val work = IdentifiedWork(
        canonicalId = "t83tggem",
        title = Some("Tired of troubling tests"),
        sourceIdentifier = SourceIdentifier(
          identifierScheme = IdentifierSchemes.miroImageNumber,
          ontologyType = "work",
          value = "T0083000"
        ),
        version = 1
      )

      val elasticsearchHitJson = s"""{
        "_index": "zagbdjgf",
        "_type": "work",
        "_id": "${work.canonicalId}",
        "_score": 1,
        "_source": ${toJson(work).get}
      }"""

      val futureDisplayWork: Future[DisplayWork] = Source
        .single(elasticsearchHitJson)
        .via(flow)
        .runWith(Sink.head)

      whenReady(futureDisplayWork) { displayWork =>
        displayWork shouldBe DisplayWork(work, includes = includes)
      }
    }
  }
  //
  // it("creates a SierraRecord from an item") {
  //   withRecordWrapperFlow {
  //     case (wrapperFlow, actorSystem) =>
  //       implicit val system: ActorSystem = actorSystem
  //       implicit val materializer: Materializer =
  //         ActorMaterializer()(actorSystem)
  //
  //       val id = "400004"
  //       val updatedDate = "2014-04-14T14:14:14Z"
  //       val json = parse(s"""
  //       |{
  //       | "id": "$id",
  //       | "updatedDate": "$updatedDate",
  //       | "bibIds": ["4", "44", "444", "4444"]
  //       |}
  //     """.stripMargin).right.get
  //
  //       val expectedRecord = SierraRecord(
  //         id = id,
  //         data = json.noSpaces,
  //         modifiedDate = updatedDate
  //       )
  //
  //       val futureRecord = Source
  //         .single(json)
  //         .via(wrapperFlow)
  //         .runWith(Sink.head)
  //
  //       whenReady(futureRecord) { sierraRecord =>
  //         sierraRecord shouldBe expectedRecord
  //       }
  //   }
  // }
  //
  // it("is able to handle deleted bibs") {
  //   withRecordWrapperFlow {
  //     case (wrapperFlow, actorSystem) =>
  //       implicit val system: ActorSystem = actorSystem
  //       implicit val materializer: Materializer =
  //         ActorMaterializer()(actorSystem)
  //
  //       val id = "1357947"
  //       val deletedDate = "2014-01-31"
  //       val json = parse(s"""{
  //                         |  "id" : "$id",
  //                         |  "deletedDate" : "$deletedDate",
  //                         |  "deleted" : true
  //                         |}""".stripMargin).right.get
  //
  //       val expectedRecord = SierraRecord(
  //         id = id,
  //         data = json.noSpaces,
  //         modifiedDate = s"${deletedDate}T00:00:00Z"
  //       )
  //
  //       val futureRecord = Source
  //         .single(json)
  //         .via(wrapperFlow)
  //         .runWith(Sink.head)
  //
  //       whenReady(futureRecord) { sierraRecord =>
  //         sierraRecord shouldBe expectedRecord
  //       }
  //   }
  // }
  //
  // it("fails the stream if the record contains invalid JSON") {
  //   withRecordWrapperFlow {
  //     case (wrapperFlow, actorSystem) =>
  //       implicit val system: ActorSystem = actorSystem
  //       implicit val materializer: Materializer =
  //         ActorMaterializer()(actorSystem)
  //
  //       val invalidSierraJson = parse(s"""{
  //       | "missing": ["id", "updatedDate"],
  //       | "reason": "This JSON will not pass!",
  //       |  "comment": "XML is coming!"
  //       |}""".stripMargin).right.get
  //
  //       val futureUnit = Source
  //         .single(invalidSierraJson)
  //         .via(wrapperFlow)
  //         .runWith(Sink.head)
  //
  //       whenReady(futureUnit.failed) { _ =>
  //         true shouldBe true
  //       }
  //   }
  // }
}<commit_msg>Add the remaining tests for the flow<commit_after>package uk.ac.wellcome.platform.snapshot_convertor.flow

import akka.actor.ActorSystem
import akka.stream.{ActorMaterializer, Materializer}
import akka.stream.scaladsl.{Sink, Source}
import org.scalatest.concurrent.ScalaFutures
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.display.models.DisplayWork
import uk.ac.wellcome.exceptions.GracefulFailureException
import uk.ac.wellcome.models.{IdentifiedWork, IdentifierSchemes, SourceIdentifier, WorksIncludes}
import uk.ac.wellcome.test.fixtures.AkkaFixtures
import uk.ac.wellcome.test.utils.ExtendedPatience
import uk.ac.wellcome.utils.JsonUtil._

import scala.concurrent.{ExecutionContextExecutor, Future}

class ElasticsearchHitToDisplayWorkFlowTest
    extends FunSpec
    with Matchers
    with AkkaFixtures
    with ScalaFutures
    with ExtendedPatience {

  val includes = WorksIncludes(
    identifiers = true,
    thumbnail = true,
    items = true
  )

  it("creates a DisplayWork from a single hit") {
    withActorSystem { actorSystem =>
      implicit val executionContext: ExecutionContextExecutor =
        actorSystem.dispatcher
      implicit val system: ActorSystem = actorSystem
      implicit val materializer: Materializer =
        ActorMaterializer()(actorSystem)

      val flow = ElasticsearchHitToDisplayWorkFlow()

      val work = IdentifiedWork(
        canonicalId = "t83tggem",
        title = Some("Tired of troubling tests"),
        sourceIdentifier = SourceIdentifier(
          identifierScheme = IdentifierSchemes.miroImageNumber,
          ontologyType = "work",
          value = "T0083000"
        ),
        version = 1
      )

      val elasticsearchHitJson = s"""{
        "_index": "zagbdjgf",
        "_type": "work",
        "_id": "${work.canonicalId}",
        "_score": 1,
        "_source": ${toJson(work).get}
      }"""

      val futureDisplayWork: Future[DisplayWork] = Source
        .single(elasticsearchHitJson)
        .via(flow)
        .runWith(Sink.head)

      whenReady(futureDisplayWork) { displayWork =>
        displayWork shouldBe DisplayWork(work, includes = includes)
      }
    }
  }

  it("returns a failed Future if it gets invalid JSON") {
    withActorSystem { actorSystem =>
      implicit val executionContext: ExecutionContextExecutor =
        actorSystem.dispatcher
      implicit val system: ActorSystem = actorSystem
      implicit val materializer: Materializer =
        ActorMaterializer()(actorSystem)

      val flow = ElasticsearchHitToDisplayWorkFlow()

      val elasticsearchHitJson = s"""MARC?XML RAARGH NOTJSON"""

      val future = Source
        .single(elasticsearchHitJson)
        .via(flow)
        .runWith(Sink.head)

      whenReady(future.failed) { result =>
        result shouldBe a[GracefulFailureException]
      }
    }
  }

  it("returns a failed Future if it gets valid JSON but _source isn't a Work") {
    withActorSystem { actorSystem =>
      implicit val executionContext: ExecutionContextExecutor =
        actorSystem.dispatcher
      implicit val system: ActorSystem = actorSystem
      implicit val materializer: Materializer =
        ActorMaterializer()(actorSystem)

      val flow = ElasticsearchHitToDisplayWorkFlow()

      val elasticsearchHitJson = s"""{
        "_index": "rd8a35zw",
        "_type": "work",
        "_id": "ndpwrqer",
        "_score": 1,
        "_source": {"foo": "bar", "baz": "bat"}
      }"""

      val future = Source
        .single(elasticsearchHitJson)
        .via(flow)
        .runWith(Sink.head)

      whenReady(future.failed) { result =>
        result shouldBe a[GracefulFailureException]
      }
    }
  }
}
{'fraction_non_alphanumeric': 0.1154157511275587, 'fraction_numerical': 0.00890482248178559, 'mean_word_length': 2.2907153729071537, 'pattern_counts': {'":': 23, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572472', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2544, 'n_words': 531}