<commit_before>module "catalogue_pipeline" {
  source = "pipeline"

  namespace = "catalogue_pipeline"

  transformer_miro_container_image   = "${local.transformer_miro_container_image}"
  transformer_sierra_container_image = "${local.transformer_sierra_container_image}"
  recorder_container_image           = "${local.recorder_container_image}"
  matcher_container_image            = "${local.matcher_container_image}"
  merger_container_image             = "${local.merger_container_image}"
  id_minter_container_image          = "${local.id_minter_container_image}"
  ingestor_container_image           = "${local.ingestor_container_image}"

  subnets = ["${local.private_subnets}"]
  vpc_id  = "${local.vpc_id}"

  account_id = "${data.aws_caller_identity.current.account_id}"

  vhs_miro_read_policy = "${local.vhs_miro_read_policy}"

  vhs_sierra_read_policy = "${local.vhs_sierra_read_policy}"

  aws_region      = "${var.aws_region}"
  messages_bucket = "${aws_s3_bucket.messages.id}"
  infra_bucket    = "${var.infra_bucket}"

  index_v1 = "v1-2018-08-30-parallel-scan-reindexer"
  index_v2 = "v2-2018-08-30-parallel-scan-reindexer"

  rds_access_security_group_id = "${local.rds_access_security_group_id}"

  identifiers_rds_cluster_password = "${local.identifiers_rds_cluster_password}"
  identifiers_rds_cluster_username = "${local.identifiers_rds_cluster_username}"
  identifiers_rds_cluster_port     = "${local.identifiers_rds_cluster_port}"
  identifiers_rds_cluster_host     = "${local.identifiers_rds_cluster_host}"

  es_cluster_credentials = "${var.es_cluster_credentials}"
  dlq_alarm_arn          = "${local.dlq_alarm_arn}"
  lambda_error_alarm_arn = "${local.lambda_error_alarm_arn}"

  miro_adapter_topic_names = [
    "${local.miro_reindexer_topic_name}",
  ]

  sierra_adapter_topic_names = [
    "${local.sierra_reindexer_topic_name}",
    "${local.sierra_merged_bibs_topic_name}",
    "${local.sierra_merged_items_topic_name}",
  ]

  service_egress_security_group_id = "${module.service_egress_security_group.sg_id}"

  vhs_bucket_name = "${aws_s3_bucket.vhs_bucket.id}"
}<commit_msg>New pipeline name and new index<commit_after>module "catalogue_pipeline" {
  source = "pipeline"

  namespace = "catalogue_pipeline_ms2"

  transformer_miro_container_image   = "${local.transformer_miro_container_image}"
  transformer_sierra_container_image = "${local.transformer_sierra_container_image}"
  recorder_container_image           = "${local.recorder_container_image}"
  matcher_container_image            = "${local.matcher_container_image}"
  merger_container_image             = "${local.merger_container_image}"
  id_minter_container_image          = "${local.id_minter_container_image}"
  ingestor_container_image           = "${local.ingestor_container_image}"

  subnets = ["${local.private_subnets}"]
  vpc_id  = "${local.vpc_id}"

  account_id = "${data.aws_caller_identity.current.account_id}"

  vhs_miro_read_policy = "${local.vhs_miro_read_policy}"

  vhs_sierra_read_policy = "${local.vhs_sierra_read_policy}"

  aws_region      = "${var.aws_region}"
  messages_bucket = "${aws_s3_bucket.messages.id}"
  infra_bucket    = "${var.infra_bucket}"

  index_v1 = "v1-2018-09-13-miro-sierra-merging-take-2"
  index_v2 = "v2-2018-09-13-miro-sierra-merging-take-2"

  rds_access_security_group_id = "${local.rds_access_security_group_id}"

  identifiers_rds_cluster_password = "${local.identifiers_rds_cluster_password}"
  identifiers_rds_cluster_username = "${local.identifiers_rds_cluster_username}"
  identifiers_rds_cluster_port     = "${local.identifiers_rds_cluster_port}"
  identifiers_rds_cluster_host     = "${local.identifiers_rds_cluster_host}"

  es_cluster_credentials = "${var.es_cluster_credentials}"
  dlq_alarm_arn          = "${local.dlq_alarm_arn}"
  lambda_error_alarm_arn = "${local.lambda_error_alarm_arn}"

  miro_adapter_topic_names = [
    "${local.miro_reindexer_topic_name}",
  ]

  sierra_adapter_topic_names = [
    "${local.sierra_reindexer_topic_name}",
    "${local.sierra_merged_bibs_topic_name}",
    "${local.sierra_merged_items_topic_name}",
  ]

  service_egress_security_group_id = "${module.service_egress_security_group.sg_id}"

  vhs_bucket_name = "${aws_s3_bucket.vhs_bucket.id}"
}
{'fraction_non_alphanumeric': 0.11706629055007052, 'fraction_numerical': 0.011048425011753644, 'mean_word_length': 5.446969696969697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2220686', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1495, 'n_words': 137}