<commit_before>package uk.ac.wellcome.transformer

import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration
import com.gu.scanamo.Scanamo
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.models.{CalmTransformable, SourceIdentifier, Work}
import uk.ac.wellcome.test.utils.MessageInfo
import uk.ac.wellcome.transformer.utils.TransformerFeatureTest
import uk.ac.wellcome.utils.JsonUtil

class CalmTransformerFeatureTest
    extends FunSpec
    with TransformerFeatureTest
    with Matchers {

  private val appName = "test-transformer-calm"

  override val flags: Map[String, String] = Map(
    "aws.region" -> "eu-west-1",
    "aws.dynamo.calmData.streams.appName" -> appName,
    "aws.dynamo.calmData.streams.arn" -> calmDataStreamArn,
    "aws.dynamo.calmData.tableName" -> calmDataTableName,
    "aws.sns.topic.arn" -> idMinterTopicArn,
    "aws.metrics.namespace" -> "calm-transformer"
  )
  override val kinesisClientLibConfiguration: KinesisClientLibConfiguration =
    kinesisClientLibConfiguration(appName, calmDataStreamArn)

  it("should poll the dynamo stream for calm data, transform it into unified items and push them into the id_minter SNS topic") {
    Scanamo.put(dynamoDbClient)(calmDataTableName)(
      CalmTransformable(RecordID = "RecordID1",
                        RecordType = "Collection",
                        AltRefNo = "AltRefNo1",
                        RefNo = "RefNo1",
                        data = """{"AccessStatus": ["public"]}"""))

    eventually {
      val snsMessages = listMessagesReceivedFromSNS()
      snsMessages should have size (1)
      assertSNSMessageContainsCalmDataWith(snsMessages.head, Some("public"))
    }

    Scanamo.put(dynamoDbClient)(calmDataTableName)(
      CalmTransformable(RecordID = "RecordID2",
                        RecordType = "Collection",
                        AltRefNo = "AltRefNo2",
                        RefNo = "RefNo2",
                        data = """{"AccessStatus": ["restricted"]}"""))

    eventually {
      val snsMessages = listMessagesReceivedFromSNS()
      snsMessages should have size (2)

      assertSNSMessageContainsCalmDataWith(snsMessages.head, Some("public"))
      assertSNSMessageContainsCalmDataWith(snsMessages.tail.head,
                                           Some("restricted"))
    }
  }

  private def assertSNSMessageContainsCalmDataWith(<commit_msg>Add line to trigger rebuild<commit_after>package uk.ac.wellcome.transformer

import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration
import com.gu.scanamo.Scanamo
import org.scalatest.{FunSpec, Matchers}
import uk.ac.wellcome.models.{CalmTransformable, SourceIdentifier, Work}
import uk.ac.wellcome.test.utils.MessageInfo
import uk.ac.wellcome.transformer.utils.TransformerFeatureTest
import uk.ac.wellcome.utils.JsonUtil

class CalmTransformerFeatureTest
    extends FunSpec
    with TransformerFeatureTest
    with Matchers {

  private val appName = "test-transformer-calm"

  override val flags: Map[String, String] = Map(
    "aws.region" -> "eu-west-1",
    "aws.dynamo.calmData.streams.appName" -> appName,
    "aws.dynamo.calmData.streams.arn" -> calmDataStreamArn,
    "aws.dynamo.calmData.tableName" -> calmDataTableName,
    "aws.sns.topic.arn" -> idMinterTopicArn,
    "aws.metrics.namespace" -> "calm-transformer"
  )
  override val kinesisClientLibConfiguration: KinesisClientLibConfiguration =
    kinesisClientLibConfiguration(appName, calmDataStreamArn)

  it("should poll the dynamo stream for calm data, transform it into unified items and push them into the id_minter SNS topic") {
    val a =1
    Scanamo.put(dynamoDbClient)(calmDataTableName)(
      CalmTransformable(RecordID = "RecordID1",
                        RecordType = "Collection",
                        AltRefNo = "AltRefNo1",
                        RefNo = "RefNo1",
                        data = """{"AccessStatus": ["public"]}"""))

    eventually {
      val snsMessages = listMessagesReceivedFromSNS()
      snsMessages should have size (1)
      assertSNSMessageContainsCalmDataWith(snsMessages.head, Some("public"))
    }

    Scanamo.put(dynamoDbClient)(calmDataTableName)(
      CalmTransformable(RecordID = "RecordID2",
                        RecordType = "Collection",
                        AltRefNo = "AltRefNo2",
                        RefNo = "RefNo2",
                        data = """{"AccessStatus": ["restricted"]}"""))

    eventually {
      val snsMessages = listMessagesReceivedFromSNS()
      snsMessages should have size (2)

      assertSNSMessageContainsCalmDataWith(snsMessages.head, Some("public"))
      assertSNSMessageContainsCalmDataWith(snsMessages.tail.head,
                                           Some("restricted"))
    }
  }

  private def assertSNSMessageContainsCalmDataWith(
{'fraction_non_alphanumeric': 0.09997947033463354, 'fraction_numerical': 0.00390063641962636, 'mean_word_length': 3.0264462809917356, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1223856', 'n_tokens_mistral': 1454, 'n_tokens_neox': 1407, 'n_words': 260}