<commit_before># This file defines alerts to be triggered by the server.

#
# SkiaPerf and SkiaGold
#

[[rule]]
name = "Perf Alerts"
message = "At least one perf alert has been found. Please visit https://perf.skia.org/alerts/ to triage."
query = "select value from /skiaperf.skia-perf.alerting.new.value/ limit 1"
category = "Perf"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Gold Alert (GM)"
message = "At least one untriaged GM has been found. Please visit https://gold.skia.org/ to triage."
query = "select value from /skia-gold-prod.skiacorrectness.skia-gold-prod.gold.untriaged.by_corpus.gm.value/ limit 1"
category = "Gold"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Gold Alert (SKP)"
message = "At least one untriaged SKP has been found. Please visit https://gold.skia.org/ to triage."
query = "select value from /skia-gold-prod.skiacorrectness.skia-gold-prod.gold.untriaged.by_corpus.skp.value/ limit 1"
category = "Gold"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Expired Ingores (Gold)"
message = "At least one expired ignore rule has been found. Please visit https://gold.skia.org/2/ignores to delete or extend."
query = "select value from /skia-gold-prod.skiacorrectness.skia-gold-prod.num-expired-ignore-rules.value/ limit 1"
category = "Gold"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"


[[rule]]
name = "Ingestion Failure (Perf)"
message = "At least two rounds of perf ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-perf.ingester.nano-ingest.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingestion Failure (Perf Trybot)"
message = "At least two rounds of perf trybot ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-perf.ingester.nano-trybot.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingestion Failure (Gold)"
message = "At least two rounds of gold ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-gold-prod.ingester.gold-ingest.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingore Monitoring Failure (Gold)"
message = "At least two rounds of monitoring for expired ignore rules have failed back to back."
query = "select mean(value) from /skia-gold-prod.skiacorrectness.skia-gold-prod.expired-ignore-rules-monitoring.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 200"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# SkFiddle
#

# All rules commented out until fiddle comes back online.

#[[rule]]
#name = "Skia Fiddle Prober (main page)"
#message = "The main page at http://skfiddle.com has failed."
#query = "select mean(value) from /prober.skfiddle.failure.value/ where time > now() - 10m;"
#category = "infra"
#condition = "x >= 1"
#actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
#auto-dismiss = true
#nag = "1h"
#
#[[rule]]
#name = "Skia Fiddle Prober (compile)"
#message = "A test compile at http://skfiddle.com has failed for more than 10 minutes."
#query = "select mean(value) from /prober.skfiddle_compile_bad.failure.value/ where time > now() - 10m;"
#category = "infra"
#condition = "x >= 1"
#actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
#auto-dismiss = true
#nag = "1h"
#
#[[rule]]
#name = "Skia Fiddle Prober (compile)"
#message = "A test compile at http://skfiddle.com has failed for more than 10 minutes."
#query = "select mean(value) from /prober.skfiddle_compile_good.failure.value/ where time > now() - 10m;"
#category = "infra"
#condition = "x >= 1"
#actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
#auto-dismiss = true
#nag = "1h"


# Skia Frontend
#

[[rule]]
name = "Skia Frontend (skfe-1)"
message = "One of the skia.org frontend servers has failed for more than 5 minutes"
query = "select mean(value) from /prober.skfe-1.failure.value/ where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Frontend (skfe-2)"
message = "One of the skia.org frontend servers has failed for more than 5 minutes"
query = "select mean(value) from /prober.skfe-2.failure.value/ where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Frontend (GCE Load Balancing)"
message = "Serving for skia.org has failed for more than 5 minutes"
query = "select mean(value) from prober.skiadev.failure.value where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"


#
# Disk space monitoring.
#

[[rule]]
name = "Low Disk Space (skia-docs)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-docs.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-monitoring)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-monitoring.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-monitoring /mnt/pd0)"
message = "Free space has fallen below 100GB."
query = "select mean(value) from /collectd.skia-monitoring.df-mnt-pd0.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e11"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-perf)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-perf.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-perf/mnt/pd0)"
message = "Free space has fallen below 100GB."
query = "select mean(value) from /collectd.skia-perf.df-mnt-pd0.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e11"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-push)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-push.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-skfe-1)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-skfe-1.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-skfe-2)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-skfe-2.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Status
#

[[rule]]
name = "Skia Status Prober (main page)"
message = "The main page at https://status.skia.org has failed."
query = "select mean(value) from /prober.skiastatus.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Status Prober (JSON endpoint)"
message = "The JSON endpoint at https://status.skia.org/json/skia/commits/ has failed."
query = "select mean(value) from /prober.skiastatus_json.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Status Latency (JSON endpoint)"
message = "The JSON endpoint at https://status.skia.org/json/skia/commits/ took more than 300ms to respond."
query = "select mean(value) from /prober.skiastatus_json.latency.value/ where time > now() - 10m;"
category = "infra"
condition = "x / 1000000 > 300"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Alerts
#

[[rule]]
name = "Skia Alerts Prober (main page)"
message = "The main page at https://alerts.skia.org has failed."
query = "select mean(value) from /prober.skiaalerts.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Alerts Prober (JSON endpoint)"
message = "The JSON endpoint at https://alerts.skia.org/json/alerts/ has failed."
query = "select mean(value) from /prober.skiaalerts_json.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Alerts Latency (JSON endpoint)"
message = "The JSON endpoint at https://alerts.skia.org/json/alerts/ took more than 200ms to respond."
query = "select mean(value) from /prober.skiaalerts_json.latency.value/ where time > now() - 10m;"
category = "infra"
condition = "x / 1000000 > 200"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Prober
#

[[rule]]
name = "Probe Failure (probes)"
message = "Probing has failed to run in at least 5 minutes."
query = "select mean(value) from /probeserver.skia-monitoring.probes.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 300"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Probe Failure (issue-tracker)"
message = "Ingesting issue tracker issued has failed to run in at least 30 minutes."
query = "select mean(value) from /probeserver.skia-monitoring.issue-tracker.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 1800"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"
<commit_msg>Turn off Gold SKP alerts until we have a resolution.<commit_after># This file defines alerts to be triggered by the server.

#
# SkiaPerf and SkiaGold
#

[[rule]]
name = "Perf Alerts"
message = "At least one perf alert has been found. Please visit https://perf.skia.org/alerts/ to triage."
query = "select value from /skiaperf.skia-perf.alerting.new.value/ limit 1"
category = "Perf"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Gold Alert (GM)"
message = "At least one untriaged GM has been found. Please visit https://gold.skia.org/ to triage."
query = "select value from /skia-gold-prod.skiacorrectness.skia-gold-prod.gold.untriaged.by_corpus.gm.value/ limit 1"
category = "Gold"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Expired Ingores (Gold)"
message = "At least one expired ignore rule has been found. Please visit https://gold.skia.org/2/ignores to delete or extend."
query = "select value from /skia-gold-prod.skiacorrectness.skia-gold-prod.num-expired-ignore-rules.value/ limit 1"
category = "Gold"
condition = "x > 0"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Ingestion Failure (Perf)"
message = "At least two rounds of perf ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-perf.ingester.nano-ingest.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingestion Failure (Perf Trybot)"
message = "At least two rounds of perf trybot ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-perf.ingester.nano-trybot.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingestion Failure (Gold)"
message = "At least two rounds of gold ingestion have failed back to back."
query = "select mean(value) from /ingest.skia-gold-prod.ingester.gold-ingest.gauge.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 750"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ingore Monitoring Failure (Gold)"
message = "At least two rounds of monitoring for expired ignore rules have failed back to back."
query = "select mean(value) from /skia-gold-prod.skiacorrectness.skia-gold-prod.expired-ignore-rules-monitoring.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 200"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# SkFiddle
#

[[rule]]
name = "Skia Fiddle Prober (main page)"
message = "The main page at http://skfiddle.com has failed."
query = "select mean(value) from /prober.skfiddle.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Fiddle Prober (compile)"
message = "A test compile at http://skfiddle.com has failed for more than 10 minutes."
query = "select mean(value) from /prober.skfiddle_compile_bad.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Fiddle Prober (compile)"
message = "A test compile at http://skfiddle.com has failed for more than 10 minutes."
query = "select mean(value) from /prober.skfiddle_compile_good.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Frontend
#

[[rule]]
name = "Skia Frontend (skfe-1)"
message = "One of the skia.org frontend servers has failed for more than 5 minutes"
query = "select mean(value) from /prober.skfe-1.failure.value/ where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Frontend (skfe-2)"
message = "One of the skia.org frontend servers has failed for more than 5 minutes"
query = "select mean(value) from /prober.skfe-2.failure.value/ where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Frontend (GCE Load Balancing)"
message = "Serving for skia.org has failed for more than 5 minutes"
query = "select mean(value) from prober.skiadev.failure.value where time > now() - 5m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"


#
# Disk space monitoring.
#

[[rule]]
name = "Low Disk Space (skia-docs)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-docs.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-monitoring)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-monitoring.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-monitoring /mnt/pd0)"
message = "Free space has fallen below 100GB."
query = "select mean(value) from /collectd.skia-monitoring.df-mnt-pd0.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e11"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-perf)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-perf.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-perf/mnt/pd0)"
message = "Free space has fallen below 100GB."
query = "select mean(value) from /collectd.skia-perf.df-mnt-pd0.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e11"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-push)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-push.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-skfe-1)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-skfe-1.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space (skia-skfe-2)"
message = "Free space has fallen below 1GB."
query = "select mean(value) from /collectd.skia-skfe-2.df-root.df_complex-free/ where time > now() - 5m;"
category = "infra"
condition = "x <= 1e9"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Status
#

[[rule]]
name = "Skia Status Prober (main page)"
message = "The main page at https://status.skia.org has failed."
query = "select mean(value) from /prober.skiastatus.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Status Prober (JSON endpoint)"
message = "The JSON endpoint at https://status.skia.org/json/skia/commits/ has failed."
query = "select mean(value) from /prober.skiastatus_json.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Status Latency (JSON endpoint)"
message = "The JSON endpoint at https://status.skia.org/json/skia/commits/ took more than 300ms to respond."
query = "select mean(value) from /prober.skiastatus_json.latency.value/ where time > now() - 10m;"
category = "infra"
condition = "x / 1000000 > 300"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Alerts
#

[[rule]]
name = "Skia Alerts Prober (main page)"
message = "The main page at https://alerts.skia.org has failed."
query = "select mean(value) from /prober.skiaalerts.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Alerts Prober (JSON endpoint)"
message = "The JSON endpoint at https://alerts.skia.org/json/alerts/ has failed."
query = "select mean(value) from /prober.skiaalerts_json.failure.value/ where time > now() - 10m;"
category = "infra"
condition = "x >= 1"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Skia Alerts Latency (JSON endpoint)"
message = "The JSON endpoint at https://alerts.skia.org/json/alerts/ took more than 200ms to respond."
query = "select mean(value) from /prober.skiaalerts_json.latency.value/ where time > now() - 10m;"
category = "infra"
condition = "x / 1000000 > 200"
actions = ["Email(<EMAIL>)", "Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

#
# Skia Prober
#

[[rule]]
name = "Probe Failure (probes)"
message = "Probing has failed to run in at least 5 minutes."
query = "select mean(value) from /probeserver.skia-monitoring.probes.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 300"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Probe Failure (issue-tracker)"
message = "Ingesting issue tracker issued has failed to run in at least 30 minutes."
query = "select mean(value) from /probeserver.skia-monitoring.issue-tracker.time-since-last-successful-update.value/ where time > now() - 10m"
category = "infra"
condition = "x >= 1800"
actions = ["Email(<EMAIL>)"]
auto-dismiss = true
nag = "1h"

{'fraction_non_alphanumeric': 0.16878288997248106, 'fraction_numerical': 0.018249408583981074, 'mean_word_length': 4.363542206110823, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 175, 'https://': 19, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5862822', 'n_tokens_mistral': 7804, 'n_tokens_neox': 7324, 'n_words': 2245}