<commit_before>resource "datadog_monitor" "cc_api_master_process_running" {
  name                = "${format("%s Cloud Controller API master process running", var.env)}"
  type                = "service check"
  message             = "Cloud Controller API master process is not running. Check deployment state."
  escalation_message  = "Cloud Controller API master process is still not running. Check deployment state."
  notify_no_data      = false
  require_full_window = true

  query = "${format("'process.up'.over('deploy_env:%s','process:cc_api_master').last(4).count_by_status()", var.env)}"

  thresholds {
    ok       = 1
    warning  = 2
    critical = 3
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_api_worker_process_running" {
  name                = "${format("%s Cloud Controller API worker process running", var.env)}"
  type                = "service check"
  message             = "Cloud Controller API worker process is not running. Check deployment state."
  escalation_message  = "Cloud Controller API worker process is still not running. Check deployment state."
  notify_no_data      = false
  require_full_window = true

  query = "${format("'process.up'.over('deploy_env:%s','process:cc_api_worker').last(4).count_by_status()", var.env)}"

  thresholds {
    ok       = 1
    warning  = 2
    critical = 3
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_api_healthy" {
  name                = "${format("%s Cloud Controller API healthy", var.env)}"
  type                = "service check"
  message             = "Large portion of Cloud Controller API master unhealthy. Check deployment state."
  escalation_message  = "Large portion of Cloud Controller API master still unhealthy. Check deployment state."
  no_data_timeframe   = "7"
  require_full_window = true

  query = "${format("'http.can_connect'.over('deploy_env:%s','instance:cc_endpoint').by('*').last(1).pct_by_status()", var.env)}"

  thresholds {
    critical = 50
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_failed_job_count_total_increase" {
  name                = "${format("%s Cloud Controller API failed job count", var.env)}"
  type                = "query alert"
  message             = "${format("Amount of failed jobs in Cloud Controller API grew considerably, check the API health. {{#is_alert}}%s{{/is_alert}}", var.datadog_notification_in_hours)}"
  escalation_message  = "Amount of failed jobs in Cloud Controller API still growing considerably, check the API health."
  require_full_window = false

  query = "${format("change(max(last_1m),last_30m):max:cf.cc.failed_job_count.total{deployment:%s}.rollup(avg, 30) > 5", var.env)}"

  thresholds {
    warning  = "3"
    critical = "5"<commit_msg>Add logsearch query for CC failed jobs alert<commit_after>resource "datadog_monitor" "cc_api_master_process_running" {
  name                = "${format("%s Cloud Controller API master process running", var.env)}"
  type                = "service check"
  message             = "Cloud Controller API master process is not running. Check deployment state."
  escalation_message  = "Cloud Controller API master process is still not running. Check deployment state."
  notify_no_data      = false
  require_full_window = true

  query = "${format("'process.up'.over('deploy_env:%s','process:cc_api_master').last(4).count_by_status()", var.env)}"

  thresholds {
    ok       = 1
    warning  = 2
    critical = 3
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_api_worker_process_running" {
  name                = "${format("%s Cloud Controller API worker process running", var.env)}"
  type                = "service check"
  message             = "Cloud Controller API worker process is not running. Check deployment state."
  escalation_message  = "Cloud Controller API worker process is still not running. Check deployment state."
  notify_no_data      = false
  require_full_window = true

  query = "${format("'process.up'.over('deploy_env:%s','process:cc_api_worker').last(4).count_by_status()", var.env)}"

  thresholds {
    ok       = 1
    warning  = 2
    critical = 3
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_api_healthy" {
  name                = "${format("%s Cloud Controller API healthy", var.env)}"
  type                = "service check"
  message             = "Large portion of Cloud Controller API master unhealthy. Check deployment state."
  escalation_message  = "Large portion of Cloud Controller API master still unhealthy. Check deployment state."
  no_data_timeframe   = "7"
  require_full_window = true

  query = "${format("'http.can_connect'.over('deploy_env:%s','instance:cc_endpoint').by('*').last(1).pct_by_status()", var.env)}"

  thresholds {
    critical = 50
  }

  tags = ["deployment:${var.env}", "service:${var.env}_monitors", "job:api"]
}

resource "datadog_monitor" "cc_failed_job_count_total_increase" {
  name                = "${format("%s Cloud Controller API failed job count", var.env)}"
  type                = "query alert"
  message             = "${format("Amount of failed jobs in Cloud Controller API grew considerably. See logsearch: '@source.component:cloud_controller_worker AND @level:ERROR' {{#is_alert}}%s{{/is_alert}}", var.datadog_notification_in_hours)}"
  escalation_message  = "Amount of failed jobs in Cloud Controller API still growing considerably. See logsearch: '@source.component:cloud_controller_worker AND @level:ERROR'"
  require_full_window = false

  query = "${format("change(max(last_1m),last_30m):max:cf.cc.failed_job_count.total{deployment:%s}.rollup(avg, 30) > 5", var.env)}"

  thresholds {
    warning  = "3"
    critical = "5"
{'fraction_non_alphanumeric': 0.12908633696563285, 'fraction_numerical': 0.006705783738474434, 'mean_word_length': 3.5403348554033487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2872911', 'n_tokens_mistral': 1833, 'n_tokens_neox': 1750, 'n_words': 464}