<commit_before># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from itertools import chain


# TODO(paul): I can't believe Python doesn't have one of these
def map_concat(func, items):
    # flatten a list-of-lists
    return list(chain.from_iterable(map(func, items)))


class BaseMetric(object):

    def __init__(self, name, labels=[]):
        self.name = name
        self.labels = labels  # OK not to clone as we never write it

    def dimension(self):
        return len(self.labels)

    def is_scalar(self):
        return not len(self.labels)

    def _render_labelvalue(self, value):
        # TODO: some kind of value escape
        return '"%s"' % (value)

    def _render_key(self, values):
        if self.is_scalar():
            return ""
        return "{%s}" % (
            ",".join(["%s=%s" % (k, self._render_labelvalue(v))
                      for k, v in zip(self.labels, values)])
        )


class CounterMetric(BaseMetric):
    """The simplest kind of metric; one that stores a monotonically-increasing
    value that counts events or running totals.

    Example use cases for Counters:
    - Number of requests processed
    - Number of items that were inserted into a queue
    - Total amount of data that a system has processed
    Counters can only go up (and be reset when the process restarts).
    """

    def __init__(self, *args, **kwargs):
        super(CounterMetric, self).__init__(*args, **kwargs)

        self.counts = {}

        # Scalar metrics are never empty
        if self.is_scalar():
            self.counts[()] = 0.

    def inc_by(self, incr, *values):
        if len(values) != self.dimension():
            raise ValueError(
                "Expected as many values to inc() as labels (%d)" % (self.dimension())
            )

        # TODO: should assert that the tag values are all strings

        if values not in self.counts:
            self.counts[values] = incr
        else:
            self.counts[values] += incr

    def inc(self, *values):
        self.inc_by(1, *values)

    def render_item(self, k):
        return ["%s%s %.12g" % (self.name, self._render_key(k), self.counts[k])]

    def render(self):
        return map_concat(self.render_item, sorted(self.counts.keys()))

<commit_msg>Add some comments to metrics classes<commit_after># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from itertools import chain


# TODO(paul): I can't believe Python doesn't have one of these
def map_concat(func, items):
    # flatten a list-of-lists
    return list(chain.from_iterable(map(func, items)))


class BaseMetric(object):
    """Base class for metrics which report a single value per label set
    """

    def __init__(self, name, labels=[]):
        """
        Args:
            name (str): principal name for this metric
            labels (list(str)): names of the labels which will be reported
                for this metric
        """
        self.name = name
        self.labels = labels  # OK not to clone as we never write it

    def dimension(self):
        return len(self.labels)

    def is_scalar(self):
        return not len(self.labels)

    def _render_labelvalue(self, value):
        # TODO: escape backslashes, quotes and newlines
        return '"%s"' % (value)

    def _render_key(self, values):
        if self.is_scalar():
            return ""
        return "{%s}" % (
            ",".join(["%s=%s" % (k, self._render_labelvalue(v))
                      for k, v in zip(self.labels, values)])
        )

    def render(self):
        """Render this metric

        Each metric is rendered as:

            name{label1="val1",label2="val2"} value

        https://prometheus.io/docs/instrumenting/exposition_formats/#text-format-details

        Returns:
            iterable[str]: rendered metrics
        """
        raise NotImplementedError()


class CounterMetric(BaseMetric):
    """The simplest kind of metric; one that stores a monotonically-increasing
    value that counts events or running totals.

    Example use cases for Counters:
    - Number of requests processed
    - Number of items that were inserted into a queue
    - Total amount of data that a system has processed
    Counters can only go up (and be reset when the process restarts).
    """

    def __init__(self, *args, **kwargs):
        super(CounterMetric, self).__init__(*args, **kwargs)

        # dict[list[str]]: value for each set of label values. the keys are the
        # label values, in the same order as the labels in self.labels.
        #
        # (if the metric is a scalar, the (single) key is the empty list).
        self.counts = {}

        # Scalar metrics are never empty
        if self.is_scalar():
            self.counts[()] = 0.

    def inc_by(self, incr, *values):
        if len(values) != self.dimension():
            raise ValueError(
                "Expected as many values to inc() as labels (%d)" % (self.dimension())
            )

        # TODO: should assert that the tag values are all strings

        if values not in self.counts:
            self.counts[values] = incr
        else:
            self.counts[values] += incr

    def inc(self, *values):
        self.inc_by(1, *values)

    def render_item(self, k):
        return ["%s%s %.12g" % (self.name, self._render_key(k), self.counts[k])]

    def render(self):
        return map_concat(self.render_item, sorted(self.counts.keys()))


{'fraction_non_alphanumeric': 0.10303770416730271, 'fraction_numerical': 0.005800641123492597, 'mean_word_length': 2.507494646680942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4195728', 'n_tokens_mistral': 1895, 'n_tokens_neox': 1806, 'n_words': 691}