<commit_before>package uk.ac.wellcome.platform.snapshot_generator.source

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Sink, Source}
import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures}
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.elasticsearch.test.fixtures.ElasticsearchFixtures
import uk.ac.wellcome.models.work.generators.WorksGenerators
import uk.ac.wellcome.models.work.internal.IdentifiedWork
import uk.ac.wellcome.test.fixtures.{Akka, TestWith}

class ElasticsearchSourceTest
    extends FunSpec
    with Matchers
    with ScalaFutures
    with IntegrationPatience
    with Akka
    with ElasticsearchFixtures
    with WorksGenerators {

  it("outputs the entire content of the index") {
    withActorSystem { actorSystem =>
      withMaterializer(actorSystem) { implicit materializer =>
        withLocalElasticsearchIndex { indexName =>
          val works = createIdentifiedWorks(count = 10)
          insertIntoElasticsearch(indexName, works: _*)

          withSource(actorSystem, indexName) { source =>
            val future = source.runWith(Sink.seq)

            whenReady(future) { result =>
              result should contain theSameElementsAs works
            }
          }
        }
      }
    }
  }

  it("filters non visible works") {
    withActorSystem { actorSystem =>
      withMaterializer(actorSystem) { implicit materializer =>
        withLocalElasticsearchIndex { indexName =>
          val visibleWorks = createIdentifiedWorks(count = 10)
          val invisibleWorks = createIdentifiedInvisibleWorks(count = 3)

          val works = visibleWorks ++ invisibleWorks
          insertIntoElasticsearch(indexName, works: _*)

          withSource(actorSystem, indexName) { source =>
            val future = source.runWith(Sink.seq)

            whenReady(future) { result =>
              result should contain theSameElementsAs visibleWorks
            }
          }
        }
      }
    }
  }

  private def withSource[R](actorSystem: ActorSystem, indexName: String)(
    testWith: TestWith[Source[IdentifiedWork, NotUsed], R]): R = {
    val source = ElasticsearchWorksSource(
      elasticClient = elasticClient,
      indexName = indexName,
      documentType = documentType)(actorSystem)
    testWith(source)
  }

}<commit_msg>We can pass ActorSystem implicitly here<commit_after>package uk.ac.wellcome.platform.snapshot_generator.source

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.scaladsl.{Sink, Source}
import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures}
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.elasticsearch.test.fixtures.ElasticsearchFixtures
import uk.ac.wellcome.models.work.generators.WorksGenerators
import uk.ac.wellcome.models.work.internal.IdentifiedWork
import uk.ac.wellcome.test.fixtures.{Akka, TestWith}

class ElasticsearchSourceTest
    extends FunSpec
    with Matchers
    with ScalaFutures
    with IntegrationPatience
    with Akka
    with ElasticsearchFixtures
    with WorksGenerators {

  it("outputs the entire content of the index") {
    withActorSystem { implicit actorSystem =>
      withMaterializer(actorSystem) { implicit materializer =>
        withLocalElasticsearchIndex { indexName =>
          val works = createIdentifiedWorks(count = 10)
          insertIntoElasticsearch(indexName, works: _*)

          withSource(indexName) { source =>
            val future = source.runWith(Sink.seq)

            whenReady(future) { result =>
              result should contain theSameElementsAs works
            }
          }
        }
      }
    }
  }

  it("filters non visible works") {
    withActorSystem { implicit actorSystem =>
      withMaterializer(actorSystem) { implicit materializer =>
        withLocalElasticsearchIndex { indexName =>
          val visibleWorks = createIdentifiedWorks(count = 10)
          val invisibleWorks = createIdentifiedInvisibleWorks(count = 3)

          val works = visibleWorks ++ invisibleWorks
          insertIntoElasticsearch(indexName, works: _*)

          withSource(indexName) { source =>
            val future = source.runWith(Sink.seq)

            whenReady(future) { result =>
              result should contain theSameElementsAs visibleWorks
            }
          }
        }
      }
    }
  }

  private def withSource[R](indexName: String)(
    testWith: TestWith[Source[IdentifiedWork, NotUsed], R])(
    implicit actorSystem: ActorSystem): R = {
    val source = ElasticsearchWorksSource(
      elasticClient = elasticClient,
      indexName = indexName,
      documentType = documentType)
    testWith(source)
  }
}
{'fraction_non_alphanumeric': 0.0799914675767918, 'fraction_numerical': 0.0021331058020477816, 'mean_word_length': 2.9940374787052813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6653857', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1272, 'n_words': 270}