<commit_before>import sys
import time
import gevent
from copy import copy
import math
import functools

from exception import InterruptLocust
from collections import deque
import events

class RequestStatsAdditionError(Exception):
    pass
        

class RequestStats(object):
    requests = {}
    request_observers = []
    total_num_requests = 0
    global_max_requests = None
    global_last_request_timestamp = None
    global_start_time = None
    errors = {}

    def __init__(self, name):
        self.name = name
        self.num_reqs_per_sec = {}
        self.last_request_timestamp = 0
        self.reset()
    
    @classmethod
    def reset_all(cls):
        cls.total_num_requests = 0
        for name, stats in cls.requests.iteritems():
            stats.reset()
        cls.errors = {}
    
    def reset(self):
        self.start_time = time.time()
        self.num_reqs = 0
        self.num_failures = 0
        self.total_response_time = 0
        self.response_times = {}
        self._min_response_time = None
        self.max_response_time = 0
        self.last_request_timestamp = int(time.time())
        self.num_reqs_per_sec = {}
        self.total_content_length = 0

    def log(self, response_time, content_length):
        RequestStats.total_num_requests += 1

        self.num_reqs += 1
        self.total_response_time += response_time
        
        t = int(time.time())
        self.num_reqs_per_sec[t] = self.num_reqs_per_sec.setdefault(t, 0) + 1
        self.last_request_timestamp = t
        RequestStats.global_last_request_timestamp = t

        if self._min_response_time is None:
            self._min_response_time = response_time
            
        self._min_response_time = min(self._min_response_time, response_time)
        self.max_response_time = max(self.max_response_time, response_time)
        
        # to avoid to much data that has to be transfered to the master node when
        # running in distributed mode, we save the response time rounded in a dict
        # so that 147 becomes 150, 3432 becomes 3400 and 58760 becomes 59000
        if response_time < 100:
            rounded_response_time = response_time
        elif response_time < 1000:
            rounded_response_time = int(round(response_time, -1))
        elif response_time < 10000:
            rounded_response_time = int(round(response_time, -2))
        else:
            rounded_response_time = int(round(response_time, -3))
        # increase request count for the rounded key in response time dict
        self.response_times.setdefault(rounded_response_time, 0)
        self.response_times[rounded_response_time] += 1
        
        # increase total content-length
        self.total_content_length += content_length
    
    def log_error(self, error):
        self.num_failures += 1
        key = "%r: %s" % (error, repr(str(error)))
        RequestStats.errors.setdefault(key, 0)
        RequestStats.errors[key] += 1
        
    @property
    def min_response_time(self):
        if self._min_response_time is None:
            return 0
        return self._min_response_time
    
    @property
    def avg_response_time(self):
        try:
            return float(self.total_response_time) / self.num_reqs
        except ZeroDivisionError:
            return 0
    
    @property
    def median_response_time(self):
        if not self.response_times:
            return 0
        
        def median(total, count):
            """
            total is the number of requests made
            count is a dict {response_time: count}
            """
            pos = (total - 1) / 2
            for k in sorted(count.iterkeys()):
                if pos < count[k]:
                    return k
                pos -= count[k]
        
        return median(self.num_reqs, self.response_times)
    
    @property
    def current_rps(self):
        if self.global_last_request_timestamp is None:
            return 0
        slice_start_time = max(self.global_last_request_timestamp - 12, int(self.global_start_time or 0))
        
        reqs = [self.num_reqs_per_sec.get(t, 0) for t in range(slice_start_time, self.global_last_request_timestamp-2)]
        return avg(reqs)

    @property
    def total_rps(self):
        if not RequestStats.global_last_request_timestamp:
            return 0.0
        
        return self.num_reqs / max(RequestStats.global_last_request_timestamp - RequestStats.global_start_time, 1)
    
    @property
    def avg_content_length(self):
        try:
            return self.total_content_length / self.num_reqs
        except ZeroDivisionError:
            return 0

    def __add__(self, other):
        #if self.name != other.name:
        #    raise RequestStatsAdditionError("Trying to add two RequestStats objects of different names (%s and %s)" % (self.name, other.name))
        
        new = RequestStats(self.name)
        new.last_request_timestamp = max(self.last_request_timestamp, other.last_request_timestamp)
        new.start_time = min(self.start_time, other.start_time)
        
        new.num_reqs = self.num_reqs + other.num_reqs
        new.num_failures = self.num_failures + other.num_failures
        new.total_response_time = self.total_response_time + other.total_response_time
        new.max_response_time = max(self.max_response_time, other.max_response_time)
        new._min_response_time = min(self._min_response_time, other._min_response_time) or other._min_response_time
        new.total_content_length = self.total_content_length + other.total_content_length        
        
        def merge_dict_add(d1, d2):
            """Merge two dicts by adding the values from each dict"""
            merged = {}
            for key in set(d1.keys() + d2.keys()):
                merged[key] = d1.get(key, 0) + d2.get(key, 0)
            return merged
        
        new.num_reqs_per_sec = merge_dict_add(self.num_reqs_per_sec, other.num_reqs_per_sec)
        new.response_times = merge_dict_add(self.response_times, other.response_times)
        return new
    
    def get_stripped_report(self):
        report = copy(self)
        self.reset()
        return report
    
    def to_dict(self):
        return {
            'num_reqs': self.num_reqs,
            'num_failures': self.num_failures,
            'avg': self.avg_response_time,
            'min': self.min_response_time,
            'max': self.max_response_time,
            'current_req_per_sec': self.current_rps
        }

    def __str__(self):
        try:<commit_msg>Optimized the summing of RequestStats objects which will increase stats reporting performance.<commit_after>import sys
import time
import gevent
from copy import copy
import math
import functools

from exception import InterruptLocust
from collections import deque
import events

class RequestStatsAdditionError(Exception):
    pass
        

class RequestStats(object):
    requests = {}
    request_observers = []
    total_num_requests = 0
    global_max_requests = None
    global_last_request_timestamp = None
    global_start_time = None
    errors = {}

    def __init__(self, name):
        self.name = name
        self.num_reqs_per_sec = {}
        self.last_request_timestamp = 0
        self.reset()
    
    @classmethod
    def reset_all(cls):
        cls.total_num_requests = 0
        for name, stats in cls.requests.iteritems():
            stats.reset()
        cls.errors = {}
    
    def reset(self):
        self.start_time = time.time()
        self.num_reqs = 0
        self.num_failures = 0
        self.total_response_time = 0
        self.response_times = {}
        self._min_response_time = None
        self.max_response_time = 0
        self.last_request_timestamp = int(time.time())
        self.num_reqs_per_sec = {}
        self.total_content_length = 0

    def log(self, response_time, content_length):
        RequestStats.total_num_requests += 1

        self.num_reqs += 1
        self.total_response_time += response_time
        
        t = int(time.time())
        self.num_reqs_per_sec[t] = self.num_reqs_per_sec.setdefault(t, 0) + 1
        self.last_request_timestamp = t
        RequestStats.global_last_request_timestamp = t

        if self._min_response_time is None:
            self._min_response_time = response_time
            
        self._min_response_time = min(self._min_response_time, response_time)
        self.max_response_time = max(self.max_response_time, response_time)
        
        # to avoid to much data that has to be transfered to the master node when
        # running in distributed mode, we save the response time rounded in a dict
        # so that 147 becomes 150, 3432 becomes 3400 and 58760 becomes 59000
        if response_time < 100:
            rounded_response_time = response_time
        elif response_time < 1000:
            rounded_response_time = int(round(response_time, -1))
        elif response_time < 10000:
            rounded_response_time = int(round(response_time, -2))
        else:
            rounded_response_time = int(round(response_time, -3))
        # increase request count for the rounded key in response time dict
        self.response_times.setdefault(rounded_response_time, 0)
        self.response_times[rounded_response_time] += 1
        
        # increase total content-length
        self.total_content_length += content_length
    
    def log_error(self, error):
        self.num_failures += 1
        key = "%r: %s" % (error, repr(str(error)))
        RequestStats.errors.setdefault(key, 0)
        RequestStats.errors[key] += 1
        
    @property
    def min_response_time(self):
        if self._min_response_time is None:
            return 0
        return self._min_response_time
    
    @property
    def avg_response_time(self):
        try:
            return float(self.total_response_time) / self.num_reqs
        except ZeroDivisionError:
            return 0
    
    @property
    def median_response_time(self):
        if not self.response_times:
            return 0
        
        def median(total, count):
            """
            total is the number of requests made
            count is a dict {response_time: count}
            """
            pos = (total - 1) / 2
            for k in sorted(count.iterkeys()):
                if pos < count[k]:
                    return k
                pos -= count[k]
        
        return median(self.num_reqs, self.response_times)
    
    @property
    def current_rps(self):
        if self.global_last_request_timestamp is None:
            return 0
        slice_start_time = max(self.global_last_request_timestamp - 12, int(self.global_start_time or 0))
        
        reqs = [self.num_reqs_per_sec.get(t, 0) for t in range(slice_start_time, self.global_last_request_timestamp-2)]
        return avg(reqs)

    @property
    def total_rps(self):
        if not RequestStats.global_last_request_timestamp:
            return 0.0
        
        return self.num_reqs / max(RequestStats.global_last_request_timestamp - RequestStats.global_start_time, 1)
    
    @property
    def avg_content_length(self):
        try:
            return self.total_content_length / self.num_reqs
        except ZeroDivisionError:
            return 0

    def __add__(self, other):
        #if self.name != other.name:
        #    raise RequestStatsAdditionError("Trying to add two RequestStats objects of different names (%s and %s)" % (self.name, other.name))
        
        new = RequestStats(self.name)
        new.last_request_timestamp = max(self.last_request_timestamp, other.last_request_timestamp)
        new.start_time = min(self.start_time, other.start_time)
        
        new.num_reqs = self.num_reqs + other.num_reqs
        new.num_failures = self.num_failures + other.num_failures
        new.total_response_time = self.total_response_time + other.total_response_time
        new.max_response_time = max(self.max_response_time, other.max_response_time)
        new._min_response_time = min(self._min_response_time, other._min_response_time) or other._min_response_time
        new.total_content_length = self.total_content_length + other.total_content_length        
        
        def merge_dict_add(d1, d2):
            """Merge two dicts by adding the values from each dict"""
            merged = copy(d1)
            for key in set(d2.keys()):
                merged[key] = d1.get(key, 0) + d2.get(key, 0)
            return merged
        
        new.num_reqs_per_sec = merge_dict_add(self.num_reqs_per_sec, other.num_reqs_per_sec)
        new.response_times = merge_dict_add(self.response_times, other.response_times)
        return new
    
    def get_stripped_report(self):
        report = copy(self)
        self.reset()
        return report
    
    def to_dict(self):
        return {
            'num_reqs': self.num_reqs,
            'num_failures': self.num_failures,
            'avg': self.avg_response_time,
            'min': self.min_response_time,
            'max': self.max_response_time,
            'current_req_per_sec': self.current_rps
        }

    def __str__(self):
        try:
{'fraction_non_alphanumeric': 0.07729651824319199, 'fraction_numerical': 0.011985132367442919, 'mean_word_length': 2.2440944881889764, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5782775', 'n_tokens_mistral': 4014, 'n_tokens_neox': 3854, 'n_words': 868}