<commit_before>module "reindex_request_creator_queue" {
  source      = "git::https://github.com/wellcometrust/terraform-modules.git//sqs?ref=v10.2.3"
  queue_name  = "reindex_request_creator_queue"
  aws_region  = "${var.aws_region}"
  account_id  = "${data.aws_caller_identity.current.account_id}"
  topic_names = ["${module.reindex_jobs_topic.name}"]

  # Each reindexer message requires up to 1500 calls to DynamoDB's PutItem API
  # (one for every record in the shard).  Particularly when the reindexer has
  # just started, it tends to hit the throughput limits, and the message
  # hits the DLQ.
  #
  # The long-term fix is to rearchitect the reindexer to behave in a better
  # way, but for now we have this short-term fix:
  #
  #   - Wait 2 minutes before a message can be reprocessed.  If it hits limits
  #     on the first attempt, the table should have warmed up before it comes
  #     round again.
  #   - Retry 50 times per job, so we have to average 30 successful PutItem
  #     calls per attempt.  This seems more feasible than the default.
  #
  # TODO: Reduce these limits when we fix the reindexer.
  #
  visibility_timeout_seconds = 120

  max_receive_count = 50
  alarm_topic_arn   = "${local.dlq_alarm_arn}"
}

module "reindex_requests_queue" {<commit_msg>Bump the visibility timeout on the reindexer queue<commit_after>module "reindex_request_creator_queue" {
  source      = "git::https://github.com/wellcometrust/terraform-modules.git//sqs?ref=v10.2.3"
  queue_name  = "reindex_request_creator_queue"
  aws_region  = "${var.aws_region}"
  account_id  = "${data.aws_caller_identity.current.account_id}"
  topic_names = ["${module.reindex_jobs_topic.name}"]

  # The reindex_request_creator scans a source data table to look for
  # records that need reindexing, and sends each of them as an SNS notification
  # for the reindex_request_processor.
  #
  # If it fails midway through, it rescans and resends *the entire shard* --
  # leading to significantly more work for the processor!  If it's already
  # sent a notification for a record, but the processor hasn't reindexed
  # it yet, it gets resent.
  #
  # To that end, we have a very long visibility timeout on this queue -- so
  # once it's sent a batch of notifications, the processor has a chance to
  # pick them up before the creator sends a fresh batch.
  #
  # (tl;dr: There is a sensible reason for the timeout to be this long, and
  #  we're not just trying to evade flakiness.)
  #
  visibility_timeout_seconds = 600

  max_receive_count = 50
  alarm_topic_arn   = "${local.dlq_alarm_arn}"
}

module "reindex_requests_queue" {
{'fraction_non_alphanumeric': 0.08126669210225104, 'fraction_numerical': 0.01030141167493323, 'mean_word_length': 3.802197802197802, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6175264', 'n_tokens_mistral': 864, 'n_tokens_neox': 825, 'n_words': 291}