<commit_before>import json
import logging
import textwrap
import urllib
import urllib2

from mlabns.util import constants
from mlabns.util import message

# This global dict maps tool_ids to the corresponding Prometheus query that
# will return the status for that tool.
#
# IMPORTANT NOTE: When querying multiple metrics from the same exporter (e.g.,
# node_exporter), be sure that the label set for each metric is unique. The `OR`
# operator in Prometheus will exclude all metrics but one in the case where they
# have the same label set. Be cautious of metrics that are generated by the
# same exporter (e.g., blackbox_exporter).  So if, for example, another
# blackbox_exporter metric is added (i.e., another probe_success), something
# may need to be done to make sure that the label sets are unique for each
# metric, even if it means using label_replace() in these queries.
#
# The label_replace() function is necessary in various places because those
# metrics have no concept of "experiment", yet this label is required for
# proper grouping of the query, and, more importantly, the output of the query
# to work as intended.  label_replace() here just manually adds the experiment
# label to every metric with a static value equal to the experiment name (e.g.,
# "ndt.iupui").
QUERIES = {
    'ndt': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_raw"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_raw_ipv6"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ssl': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_ssl"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ssl_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_ssl_ipv6"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt7': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt7"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt7_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt7_ipv6"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform-cluster", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform-cluster", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform-cluster"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'neubot': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="neubot"} OR
            probe_success{service="neubot_tls"} OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "neubot.mlab", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "neubot.mlab", "", "") != bool 1
        )
        """),
    'neubot_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="neubot_ipv6"} OR
            probe_success{service="neubot_tls_ipv6"} OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "neubot.mlab", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "neubot.mlab", "", "") != bool 1
        )
        """),
    'wehe': textwrap.dedent("""\
        min by (experiment, machine) (
            script_success{service="wehe_client"} OR
            label_replace(kube_node_spec_taint{cluster="platform-cluster", key="lame-duck"},
              "experiment", "wehe", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "wehe", "", "") != bool 1
        )
        """),
}


class Error(Exception):
    pass


class PrometheusStatusUnparseableError(Error):
    """Indicates that there was an error parsing Prometheus status information."""
<commit_msg>Fixes label values for Prometheus status queries (#254)<commit_after>import json
import logging
import textwrap
import urllib
import urllib2

from mlabns.util import constants
from mlabns.util import message

# This global dict maps tool_ids to the corresponding Prometheus query that
# will return the status for that tool.
#
# IMPORTANT NOTE: When querying multiple metrics from the same exporter (e.g.,
# node_exporter), be sure that the label set for each metric is unique. The `OR`
# operator in Prometheus will exclude all metrics but one in the case where they
# have the same label set. Be cautious of metrics that are generated by the
# same exporter (e.g., blackbox_exporter).  So if, for example, another
# blackbox_exporter metric is added (i.e., another probe_success), something
# may need to be done to make sure that the label sets are unique for each
# metric, even if it means using label_replace() in these queries.
#
# The label_replace() function is necessary in various places because those
# metrics have no concept of "experiment", yet this label is required for
# proper grouping of the query, and, more importantly, the output of the query
# to work as intended.  label_replace() here just manually adds the experiment
# label to every metric with a static value equal to the experiment name (e.g.,
# "ndt.iupui").
QUERIES = {
    'ndt': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_raw"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_raw_ipv6"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ssl': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_ssl"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt_ssl_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt_ssl_ipv6"} OR
            script_success{service="ndt5_client"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt7': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt7"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'ndt7_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="ndt7_ipv6"} OR
            label_replace(((node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"} -
              node_filesystem_avail_bytes{cluster="platform", mountpoint="/cache/data"}) /
                node_filesystem_size_bytes{cluster="platform", mountpoint="/cache/data"}),
                "experiment", "ndt.iupui", "", "") < bool 0.95 OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "ndt.iupui", "", "") != bool 1 OR
            lame_duck_experiment{cluster="platform"} != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "ndt.iupui", "", "") != bool 1
        )
        """),
    'neubot': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="neubot"} OR
            probe_success{service="neubot_tls"} OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "neubot.mlab", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "neubot.mlab", "", "") != bool 1
        )
        """),
    'neubot_ipv6': textwrap.dedent("""\
        min by (experiment, machine) (
            probe_success{service="neubot_ipv6"} OR
            probe_success{service="neubot_tls_ipv6"} OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "neubot.mlab", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "neubot.mlab", "", "") != bool 1
        )
        """),
    'wehe': textwrap.dedent("""\
        min by (experiment, machine) (
            script_success{service="wehe_client"} OR
            label_replace(kube_node_spec_taint{cluster="platform", key="lame-duck"},
              "experiment", "wehe", "", "") != bool 1 OR
            label_replace(gmx_machine_maintenance, "experiment", "wehe", "", "") != bool 1
        )
        """),
}


class Error(Exception):
    pass


class PrometheusStatusUnparseableError(Error):
    """Indicates that there was an error parsing Prometheus status information."""

{'fraction_non_alphanumeric': 0.14097744360902256, 'fraction_numerical': 0.007706766917293233, 'mean_word_length': 3.0275044158465807, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4138705', 'n_tokens_mistral': 4830, 'n_tokens_neox': 4614, 'n_words': 989}