<commit_before># archivist

resource "aws_iam_role_policy" "archivist_task_store_s3" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_store.json}"
}

resource "aws_iam_role_policy" "archivist_task_get_s3" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.ingest_get.json}"
}

resource "aws_iam_role_policy" "archivist_task_sns" {
  role   = "${module.archivist.task_role_name}"
  policy = "${module.registrar_topic.publish_policy}"
}

resource "aws_iam_role_policy" "archivist_task_sqs" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.read_from_archivist_queue.json}"
}

resource "aws_iam_role_policy" "archive_bag_vhs" {
  role   = "${module.lambda_archive_bags.role_name}"
  policy = "${module.vhs_archive_manifest.read_policy}"
}

resource "aws_iam_role_policy" "archivist_task_archive_progress_table" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}

# registrar

resource "aws_iam_role_policy" "registrar_task_get_s3" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_get.json}"
}

resource "aws_iam_role_policy" "registrar_task_vhs" {
  role   = "${module.registrar.task_role_name}"
  policy = "${module.vhs_archive_manifest.full_access_policy}"
}

resource "aws_iam_role_policy" "registrar_task_sns" {
  role   = "${module.registrar.task_role_name}"
  policy = "${module.registrar_completed_topic.publish_policy}"
}

resource "aws_iam_role_policy" "registrar_task_sqs" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.read_from_registrar_queue.json}"
}

resource "aws_iam_role_policy" "registrar_task_archive_progress_table" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}

# asset lookup lambda

data "aws_iam_policy_document" "read_from_registrar_queue" {
  statement {
    actions = [
      "sqs:DeleteMessage",
      "sqs:ReceiveMessage",
      "sqs:ChangeMessageVisibility",
    ]

    resources = [
      "${module.registrar_queue.arn}",
    ]
  }
}

resource "aws_iam_role_policy" "archive_asset_lookup_dynamo_permission" {
  role = "${module.lambda_archive_bags.role_name}"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "dynamodb:Query"
      ],
      "Resource": "${data.aws_dynamodb_table.storage_manifest.arn}"
    },
    {
      "Effect": "Allow",
      "Action": [
        "s3:HeadObject",
        "s3:GetObject"
      ],
      "Resource": "${data.aws_s3_bucket.storage_manifests.arn}/*"
    },
    {
      "Effect": "Allow",
      "Action": [
        "s3:ListBucket"
      ],
      "Resource": "${data.aws_s3_bucket.storage_manifests.arn}"
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "lambda_archive_ingest_sns" {
  role   = "${module.lambda_archive_ingest.role_name}"
  policy = "${module.archivist_topic.publish_policy}"
}

resource "aws_iam_role_policy" "lambda_archive_ingest_archive_progress_table" {
  role   = "${module.lambda_archive_ingest.role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}<commit_msg>Fix up some of the Terraform definitions<commit_after># archivist

resource "aws_iam_role_policy" "archivist_task_store_s3" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_store.json}"
}

resource "aws_iam_role_policy" "archivist_task_get_s3" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.ingest_get.json}"
}

resource "aws_iam_role_policy" "archivist_task_sns" {
  role   = "${module.archivist.task_role_name}"
  policy = "${module.registrar_topic.publish_policy}"
}

resource "aws_iam_role_policy" "archivist_task_sqs" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.read_from_archivist_queue.json}"
}

resource "aws_iam_role_policy" "archive_bag_vhs" {
  role   = "${module.lambda_archive_bags.role_name}"
  policy = "${module.vhs_archive_manifest.read_policy}"
}

resource "aws_iam_role_policy" "archivist_task_archive_progress_table" {
  role   = "${module.archivist.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}

# registrar

resource "aws_iam_role_policy" "registrar_task_get_s3" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_get.json}"
}

resource "aws_iam_role_policy" "registrar_task_vhs" {
  role   = "${module.registrar.task_role_name}"
  policy = "${module.vhs_archive_manifest.full_access_policy}"
}

resource "aws_iam_role_policy" "registrar_task_sns" {
  role   = "${module.registrar.task_role_name}"
  policy = "${module.registrar_completed_topic.publish_policy}"
}

resource "aws_iam_role_policy" "registrar_task_sqs" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.read_from_registrar_queue.json}"
}

resource "aws_iam_role_policy" "registrar_task_archive_progress_table" {
  role   = "${module.registrar.task_role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}

# asset lookup lambda

data "aws_iam_policy_document" "read_from_registrar_queue" {
  statement {
    actions = [
      "sqs:DeleteMessage",
      "sqs:ReceiveMessage",
      "sqs:ChangeMessageVisibility",
    ]

    resources = [
      "${module.registrar_queue.arn}",
    ]
  }
}

resource "aws_iam_role_policy" "archive_asset_lookup_dynamo_permission" {
  role = "${module.lambda_archive_bags.role_name}"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "dynamodb:Query"
      ],
      "Resource": "${data.aws_dynamodb_table.storage_manifest.arn}"
    },
    {
      "Effect": "Allow",
      "Action": [
        "s3:HeadObject",
        "s3:GetObject"
      ],
      "Resource": "${data.aws_s3_bucket.storage_manifests.arn}/*"
    },
    {
      "Effect": "Allow",
      "Action": [
        "s3:ListBucket"
      ],
      "Resource": "${data.aws_s3_bucket.storage_manifests.arn}"
    }
  ]
}
EOF
}

resource "aws_iam_role_policy" "lambda_archive_report_ingest_status_sns" {
  role   = "${module.lambda_archive_report_ingest_status.role_name}"
  policy = "${module.archivist_topic.publish_policy}"
}

resource "aws_iam_role_policy" "lambda_archive_report_ingest_status_archive_progress_table" {
  role   = "${module.lambda_archive_report_ingest_status.role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}

resource "aws_iam_role_policy" "lambda_archive_start_ingest_sns" {
  role   = "${module.lambda_archive_start_ingest.role_name}"
  policy = "${module.archivist_topic.publish_policy}"
}

resource "aws_iam_role_policy" "lambda_archive_start_ingest_archive_progress_table" {
  role   = "${module.lambda_archive_start_ingest.role_name}"
  policy = "${data.aws_iam_policy_document.archive_progress_table_read_write_policy.json}"
}
{'fraction_non_alphanumeric': 0.1318063365793444, 'fraction_numerical': 0.004388972706075984, 'mean_word_length': 5.066555740432612, 'pattern_counts': {'":': 22, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3185263', 'n_tokens_mistral': 2680, 'n_tokens_neox': 2617, 'n_words': 294}