<commit_before>""" test """
import time
import logging

from bzt.modules.aggregator import ResultsReader, DataPoint, KPISet
from tests import BZTestCase, r, rc, err
from tests.mocks import MockReader


class TestDefaultAggregator(BZTestCase):
    def setUp(self):
        super(TestDefaultAggregator, self).setUp()
        self.obj = ResultsReader()
        self.obj.track_percentiles = [25, 50, 75, 80, 90, 95, 99, 99.9, 100]
        self.obj.buffer_len = 1

    def test_1(self):
        obj = self.obj

        mock = MockReader()
        mock.data.append((1, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((3, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((3, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((4, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((4, "", 1, r(), r(), r(), 200, None, ''))

        obj.add_listener(mock)

        for point in mock.datapoints():
            self.assertNotEquals(0, point[DataPoint.CUMULATIVE][''][KPISet.CONCURRENCY])

        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))

        for point in mock.datapoints():
            pass

        for point in mock.datapoints(True):
            pass

        for point in mock.results:
            overall = point[DataPoint.CURRENT]['']
            self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)

    def test_speed(self):
        obj = self.obj

        mock = MockReader()

        obj.add_listener(mock)

        res = {}
        # current measurements shows ~25K samples/sec
        for cnt in (10, 100, 1000, 10000, 25000, 40000, 50000):
            for a in range(0, cnt):
                sample = (cnt, "", 1, r(1000), r(1000), r(1000), rc(), err(), '')
                mock.data.append(sample)
            before = time.time()
            for point in mock.datapoints():
                pass
            after = time.time()
            res[cnt] = after - before
            logging.info("Times: %s", res)

            while mock.results:
                point = mock.results.pop(0)
                overall = point[DataPoint.CURRENT]['']
                self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)

        for point in mock.datapoints(True):
            pass<commit_msg>debug dests<commit_after>""" test """
import time
import logging

from bzt.modules.aggregator import ResultsReader, DataPoint, KPISet
from tests import BZTestCase, r, rc, err
from tests.mocks import MockReader


class TestDefaultAggregator(BZTestCase):
    def setUp(self):
        super(TestDefaultAggregator, self).setUp()
        self.obj = ResultsReader()
        self.obj.track_percentiles = [25, 50, 75, 80, 90, 95, 99, 99.9, 100]
        self.obj.buffer_len = 1

    def test_1(self):
        obj = self.obj

        mock = MockReader()
        mock.data.append((1, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((3, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((3, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((4, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((4, "", 1, r(), r(), r(), 200, None, ''))

        obj.add_listener(mock)

        for point in mock.datapoints():
            self.assertNotEquals(0, point[DataPoint.CUMULATIVE][''][KPISet.CONCURRENCY])

        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))
        mock.data.append((2, "", 1, r(), r(), r(), 200, None, ''))

        for point in mock.datapoints():
            pass

        for point in mock.datapoints(True):
            pass

        for point in mock.results:
            overall = point[DataPoint.CURRENT]['']
            self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)

    def test_speed(self):
        obj = self.obj

        mock = MockReader()

        obj.add_listener(mock)

        res = {}
        # current measurements shows ~25K samples/sec
        for cnt in (10, 100, 1000, 10000, 25000, 40000, 50000):
            for a in range(0, cnt):
                sample = (cnt, "", 1, r(1000), r(1000), r(1000), rc(), err(), '')
                mock.data.append(sample)
            before = time.time()
            for point in mock.datapoints():
                pass
            after = time.time()
            res[cnt] = after - before
            logging.info("Times: %s", res)

            while mock.results:
                point = mock.results.pop(0)
                overall = point[DataPoint.CURRENT]['']
                self.assertTrue(len(overall[KPISet.PERCENTILES]) > 0)

        for point in mock.datapoints(True):
            pass

    def test_0buffer_scaling(self):
        obj = self.obj

        mock = MockReader()
        obj.add_listener(mock)

        # t_stamp, label, conc, r_time, con_time, latency, r_code, error, trname
        mock.min_buffer_len = 1
        mock.buffer_len = 1
        mock.buffer_scale_idx = '50.0'

        buffer_len = mock.buffer_len
        for i in range(5):
            mock.data.append((10+i, "", 1, 2, 2, 2, 200, None, ''))
        points = list(mock.datapoints())
        points = list(mock.datapoints())
        self.assertTrue(mock.buffer_len > buffer_len)
        buffer_len = mock.buffer_len

        for i in range(5):
            mock.data.append((20+i, "", 1, 3, 3, 3, 200, None, ''))
        points = list(mock.datapoints())
        points = list(mock.datapoints())
        self.assertTrue(mock.buffer_len > buffer_len)
        buffer_len = mock.buffer_len

        for i in range(5):
            mock.data.append((30+i, "", 1, 4, 4, 4, 200, None, ''))
        points = list(mock.datapoints())
        points = list(mock.datapoints())
        self.assertTrue(mock.buffer_len > buffer_len)
        buffer_len = mock.buffer_len

        for i in range(50):
            mock.data.append((40+i, "", 1, 1, 1, 1, 200, None, ''))
        points = list(mock.datapoints())
        points = list(mock.datapoints())
        self.assertTrue(mock.buffer_len <= buffer_len)
        buffer_len = mock.buffer_len

        for i in range(5):
            mock.data.append((50+i, "", 1, 1, 1, 1, 200, None, ''))
        points = list(mock.datapoints())
        points = list(mock.datapoints())
        self.assertTrue(mock.buffer_len < buffer_len)

        pass

{'fraction_non_alphanumeric': 0.1670459717796996, 'fraction_numerical': 0.043847671066605975, 'mean_word_length': 2.0992007522331924, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5311446', 'n_tokens_mistral': 2380, 'n_tokens_neox': 2117, 'n_words': 522}