<commit_before>from abc import ABCMeta, abstractmethod
from enum import unique, Enum

from common.exceptions import LogicError


class ThroughputMeasurement(metaclass=ABCMeta):
    """
    Measures request ordering throughput
    """

    @abstractmethod
    def init_time(self, start_ts):
        pass

    @abstractmethod
    def add_request(self, ordered_ts):
        pass

    @abstractmethod
    def get_throughput(self, request_time):
        return 0.0


class EMAThroughputMeasurement(ThroughputMeasurement):
    """
    Measures request ordering throughput using exponential moving average
    """

    def __init__(self, window_size=15, min_cnt=16):
        """
        Creates a throughput measurement instance.

        :param window_size: window size in seconds for each next re-calculation
        of throughput
        :param min_cnt: N - the count of last calculated values which
        contribute with significant weight in the next value being calculated
        (approximately 86% in total)
        """
        self.reqs_in_window = 0
        self.throughput = 0
        self.window_size = window_size
        self.min_cnt = min_cnt
        self.alpha = 2 / (self.min_cnt + 1)
        self.window_start_ts = None

    def init_time(self, start_ts):
        self.window_start_ts = start_ts

    def add_request(self, ordered_ts):
        self._update_time(ordered_ts)
        self.reqs_in_window += 1

    def _accumulate(self, old_accum, next_val):
        """
        Implement exponential moving average
        """
        return old_accum * (1 - self.alpha) + next_val * self.alpha

    def _process_window(self):
        self.throughput = self._accumulate(self.throughput, self.reqs_in_window / self.window_size)

    def _update_time(self, current_ts):
        while current_ts >= self.window_start_ts + self.window_size:
            self._process_window()
            self.window_start_ts += self.window_size
            self.reqs_in_window = 0

    def get_throughput(self, request_time):
        self._update_time(request_time)
        return self.throughput


class SafeStartEMAThroughputMeasurement(EMAThroughputMeasurement):
    """
    Measures request ordering throughput using exponential moving average and
    also has a safe period on start consisting of `min_cnt` windows when None
    is returned instead of a calculated value. This can be useful for leveling
    instances on start when request ordering is just beginning.
    """

    def __init__(self, *args, **kwargs):
        """
        Creates a throughput measurement instance.
        """
        super().__init__(*args, **kwargs)
        self.first_ts = None

    def init_time(self, start_ts):
        super().init_time(start_ts)
        self.first_ts = start_ts

    def get_throughput(self, request_time):
        if request_time < self.first_ts + (self.window_size * self.min_cnt):
            return None
        return super().get_throughput(request_time)
<commit_msg>INDY-1872: add more logging<commit_after>from abc import ABCMeta, abstractmethod
from enum import unique, Enum
from logging import getLogger

from common.exceptions import LogicError
logger = getLogger()


class ThroughputMeasurement(metaclass=ABCMeta):
    """
    Measures request ordering throughput
    """

    @abstractmethod
    def init_time(self, start_ts):
        pass

    @abstractmethod
    def add_request(self, ordered_ts):
        pass

    @abstractmethod
    def get_throughput(self, request_time):
        return 0.0


class EMAThroughputMeasurement(ThroughputMeasurement):
    """
    Measures request ordering throughput using exponential moving average
    """

    def __init__(self, window_size=15, min_cnt=16):
        """
        Creates a throughput measurement instance.

        :param window_size: window size in seconds for each next re-calculation
        of throughput
        :param min_cnt: N - the count of last calculated values which
        contribute with significant weight in the next value being calculated
        (approximately 86% in total)
        """
        self.reqs_in_window = 0
        self.throughput = 0
        self.window_size = window_size
        self.min_cnt = min_cnt
        self.alpha = 2 / (self.min_cnt + 1)
        self.window_start_ts = None

    def init_time(self, start_ts):
        self.window_start_ts = start_ts

    def add_request(self, ordered_ts):
        self._update_time(ordered_ts)
        self.reqs_in_window += 1

    def _accumulate(self, old_accum, next_val):
        """
        Implement exponential moving average
        """
        return old_accum * (1 - self.alpha) + next_val * self.alpha

    def _process_window(self):
        self.throughput = self._accumulate(self.throughput, self.reqs_in_window / self.window_size)

    def _update_time(self, current_ts):
        while current_ts >= self.window_start_ts + self.window_size:
            self._process_window()
            self.window_start_ts += self.window_size
            self.reqs_in_window = 0

    def get_throughput(self, request_time):
        self._update_time(request_time)
        return self.throughput


class SafeStartEMAThroughputMeasurement(EMAThroughputMeasurement):
    """
    Measures request ordering throughput using exponential moving average and
    also has a safe period on start consisting of `min_cnt` windows when None
    is returned instead of a calculated value. This can be useful for leveling
    instances on start when request ordering is just beginning.
    """

    def __init__(self, *args, **kwargs):
        """
        Creates a throughput measurement instance.
        """
        super().__init__(*args, **kwargs)
        self.first_ts = None

    def init_time(self, start_ts):
        super().init_time(start_ts)
        self.first_ts = start_ts

    def get_throughput(self, request_time):
        if request_time < self.first_ts + (self.window_size * self.min_cnt):
            return None
        logger.trace("Request time {}; First ts {}.".format(request_time, self.first_ts))
        return super().get_throughput(request_time)

{'fraction_non_alphanumeric': 0.07682227682227682, 'fraction_numerical': 0.005569205569205569, 'mean_word_length': 2.7901924270639356, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3420093', 'n_tokens_mistral': 1777, 'n_tokens_neox': 1668, 'n_words': 477}