<commit_before>import random
from threading import Thread
from time import sleep

from logger import logger
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.helpers.worker import (
    pillowfight_data_load_task,
    pillowfight_task,
)
from perfrunner.tests import PerfTest
from perfrunner.workloads.pathoGen import PathoGen
from perfrunner.workloads.revAB.__main__ import produce_ab
from perfrunner.workloads.revAB.graph import PersonIterator, generate_graph
from perfrunner.workloads.tcmalloc import WorkloadGen


class KVTest(PerfTest):

    @with_stats
    def access(self, *args):
        super().sleep()

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.hot_load()

        self.access_bg()
        self.access()

        self.report_kpi()


class MixedLatencyTest(KVTest):

    """Enable reporting of GET and SET latency."""

    COLLECTORS = {'latency': True}

    def _report_kpi(self):
        for operation in ('get', 'set'):
            self.reporter.post(
                *self.metrics.kv_latency(operation=operation)
            )


class DurabilityTest(KVTest):

    """Enable reporting of persistTo=1 and replicateTo=1 latency."""

    COLLECTORS = {'durability': True}

    def _report_kpi(self):
        for operation in ('replicate_to', 'persist_to'):
            self.reporter.post(
                *self.metrics.kv_latency(operation=operation,
                                         dbname='durability')
            )


class ReadLatencyTest(MixedLatencyTest):

    """Enable reporting of GET latency."""

    COLLECTORS = {'latency': True}

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.kv_latency(operation='get')
        )


class SubDocTest(MixedLatencyTest):

    """Enable reporting of SubDoc latency."""

    COLLECTORS = {'subdoc_latency': True}


class XATTRTest(MixedLatencyTest):

    """Enable reporting of XATTR latency."""

    COLLECTORS = {'xattr_latency': True}

    def add_xattr(self):
        access_settings = self.test_config.access_settings
        access_settings.seq_updates = True

        PerfTest.access(self, settings=access_settings)

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.add_xattr()

        self.access_bg()
        self.access()

        self.report_kpi()


class BgFetcherTest(KVTest):

    """Enable reporting of average BgFetcher wait time (disk fetches)."""

    COLLECTORS = {'net': False}

    ALL_BUCKETS = True

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_bg_wait_time()
        )


class DrainTest(KVTest):

    """Enable reporting of average disk write queue size."""

    COLLECTORS = {'net': False}

    ALL_BUCKETS = True

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_disk_write_queue()
        )


class InitialLoadTest(DrainTest):

    @with_stats
    def load(self, *args, **kwargs):
        super().load(*args, **kwargs)

    def run(self):
        self.load()

        self.report_kpi()


class BeamRssTest(KVTest):

    """Enable reporting of Erlang (beam.smp process) memory usage."""

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.max_beam_rss()
        )


class WarmupTest(PerfTest):

    """Measure the time it takes to perform cluster warm up."""

    COLLECTORS = {'net': False}

    def access(self, *args, **kwargs):
        super().sleep()

    @with_stats
    def warmup(self):
        self.remote.stop_server()
        self.remote.drop_caches()<commit_msg>Enable page cache stats in DGM cases<commit_after>import random
from threading import Thread
from time import sleep

from logger import logger
from perfrunner.helpers.cbmonitor import timeit, with_stats
from perfrunner.helpers.worker import (
    pillowfight_data_load_task,
    pillowfight_task,
)
from perfrunner.tests import PerfTest
from perfrunner.workloads.pathoGen import PathoGen
from perfrunner.workloads.revAB.__main__ import produce_ab
from perfrunner.workloads.revAB.graph import PersonIterator, generate_graph
from perfrunner.workloads.tcmalloc import WorkloadGen


class KVTest(PerfTest):

    @with_stats
    def access(self, *args):
        super().sleep()

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.hot_load()

        self.access_bg()
        self.access()

        self.report_kpi()


class MixedLatencyTest(KVTest):

    """Enable reporting of GET and SET latency."""

    COLLECTORS = {'latency': True}

    def _report_kpi(self):
        for operation in ('get', 'set'):
            self.reporter.post(
                *self.metrics.kv_latency(operation=operation)
            )


class DurabilityTest(KVTest):

    """Enable reporting of persistTo=1 and replicateTo=1 latency."""

    COLLECTORS = {'durability': True}

    def _report_kpi(self):
        for operation in ('replicate_to', 'persist_to'):
            self.reporter.post(
                *self.metrics.kv_latency(operation=operation,
                                         dbname='durability')
            )


class ReadLatencyTest(MixedLatencyTest):

    """Enable reporting of GET latency."""

    COLLECTORS = {'latency': True}

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.kv_latency(operation='get')
        )


class SubDocTest(MixedLatencyTest):

    """Enable reporting of SubDoc latency."""

    COLLECTORS = {'subdoc_latency': True}


class XATTRTest(MixedLatencyTest):

    """Enable reporting of XATTR latency."""

    COLLECTORS = {'xattr_latency': True}

    def add_xattr(self):
        access_settings = self.test_config.access_settings
        access_settings.seq_updates = True

        PerfTest.access(self, settings=access_settings)

    def run(self):
        self.load()
        self.wait_for_persistence()

        self.add_xattr()

        self.access_bg()
        self.access()

        self.report_kpi()


class BgFetcherTest(KVTest):

    """Enable reporting of average BgFetcher wait time (disk fetches)."""

    COLLECTORS = {'net': False, 'page_cache': True}

    ALL_BUCKETS = True

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_bg_wait_time()
        )


class DrainTest(KVTest):

    """Enable reporting of average disk write queue size."""

    COLLECTORS = {'net': False, 'page_cache': True}

    ALL_BUCKETS = True

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.avg_disk_write_queue()
        )


class InitialLoadTest(DrainTest):

    @with_stats
    def load(self, *args, **kwargs):
        super().load(*args, **kwargs)

    def run(self):
        self.load()

        self.report_kpi()


class BeamRssTest(KVTest):

    """Enable reporting of Erlang (beam.smp process) memory usage."""

    def _report_kpi(self):
        self.reporter.post(
            *self.metrics.max_beam_rss()
        )


class WarmupTest(PerfTest):

    """Measure the time it takes to perform cluster warm up."""

    COLLECTORS = {'net': False, 'page_cache': True}

    def access(self, *args, **kwargs):
        super().sleep()

    @with_stats
    def warmup(self):
        self.remote.stop_server()
        self.remote.drop_caches()
{'fraction_non_alphanumeric': 0.10887703460538914, 'fraction_numerical': 0.0005471207769115032, 'mean_word_length': 2.713560182833926, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2094267', 'n_tokens_mistral': 2341, 'n_tokens_neox': 2232, 'n_words': 499}