<commit_before>import os
import logging
import math
import yaml
import elasticsearch
import jinja2
import pprint
from send_email import Email

log = logging.getLogger(__name__)
pp = pprint.PrettyPrinter(indent=2)


class QueryFilter(object):
    """
    Definition of query filtering parameters
    """
    SETUP_PARAMS = ['n_db_nodes', 'n_loaders', 'n_monitor_nodes']
    SETUP_INSTANCE_PARAMS = ['instance_type_db', 'instance_type_loader', 'instance_type_monitor']

    def __init__(self, test_doc, is_gce=False):
        self.test_doc = test_doc
        self.test_type = test_doc['_type']
        self.is_gce = is_gce
        self.date_re = '/2018-*/'

    def setup_instance_params(self):
        return ['gce_' + param for param in self.SETUP_INSTANCE_PARAMS] if self.is_gce else self.SETUP_INSTANCE_PARAMS

    def filter_setup_details(self):
        setup_details = ''
        for param in self.SETUP_PARAMS + self.setup_instance_params():
            if setup_details:
                setup_details += ' AND '
            setup_details += 'setup_details.{}: {}'.format(param, self.test_doc['_source']['setup_details'][param])
        return setup_details

    def filter_test_details(self):
        test_details = 'test_details.job_name:{} '.format(
            self.test_doc['_source']['test_details']['job_name'].split('/')[0])
        test_details += self.test_cmd_details()
        test_details += ' AND test_details.time_completed: {}'.format(self.date_re)
        return test_details

    def test_cmd_details(self):
        raise NotImplementedError('Derived classes must implement this method.')

    def __call__(self, *args, **kwargs):
        try:
            return '{} AND {}'.format(self.filter_test_details(), self.filter_setup_details())
        except KeyError:
            log.exception('Expected parameters for filtering are not found , test {} {}'.format(
                self.test_type, self.test_doc['_id']))
        return None


class QueryFilterCS(QueryFilter):

    _CMD = ('cassandra-stress', )
    _PRELOAD_CMD = ('preload-cassandra-stress', )
    _PARAMS = ('command', 'cl', 'rate threads', 'schema', 'mode', 'pop', 'duration')
    _PROFILE_PARAMS = ('command', 'profile', 'ops', 'rate threads', 'duration')

    def test_details_params(self):
        return self._CMD + self._PRELOAD_CMD if \
            self.test_type.endswith('read') or self.test_type.endswith('mixed') and \
            self.test_doc['_source']['test_details'].get(self._PRELOAD_CMD[0]) else self._CMD

    def cs_params(self):
        return self._PROFILE_PARAMS if self.test_type.endswith('profiles') else self._PARAMS

    def test_cmd_details(self):
        test_details = ""
        for cs in self.test_details_params():
            for param in self.cs_params():
                if param == 'rate threads':
                    test_details += ' AND test_details.{}.rate\ threads: {}'.format(
                        cs, self.test_doc['_source']['test_details'][cs][param])
                elif param == 'duration' and cs.startswith('preload'):
                    continue
                else:
                    param_val = self.test_doc['_source']['test_details'][cs][param]
                    if param in ['profile', 'ops']:
                        param_val = "\"{}\"".format(param_val)
                    test_details += ' AND test_details.{}.{}: {}'.format(cs, param, param_val)
        return test_details


class QueryFilterScyllaBench(QueryFilter):

    _CMD = ('scylla-bench', )<commit_msg>result analyzer: filter by preload cmd params only if it exists<commit_after>import os
import logging
import math
import yaml
import elasticsearch
import jinja2
import pprint
from send_email import Email

log = logging.getLogger(__name__)
pp = pprint.PrettyPrinter(indent=2)


class QueryFilter(object):
    """
    Definition of query filtering parameters
    """
    SETUP_PARAMS = ['n_db_nodes', 'n_loaders', 'n_monitor_nodes']
    SETUP_INSTANCE_PARAMS = ['instance_type_db', 'instance_type_loader', 'instance_type_monitor']

    def __init__(self, test_doc, is_gce=False):
        self.test_doc = test_doc
        self.test_type = test_doc['_type']
        self.is_gce = is_gce
        self.date_re = '/2018-*/'

    def setup_instance_params(self):
        return ['gce_' + param for param in self.SETUP_INSTANCE_PARAMS] if self.is_gce else self.SETUP_INSTANCE_PARAMS

    def filter_setup_details(self):
        setup_details = ''
        for param in self.SETUP_PARAMS + self.setup_instance_params():
            if setup_details:
                setup_details += ' AND '
            setup_details += 'setup_details.{}: {}'.format(param, self.test_doc['_source']['setup_details'][param])
        return setup_details

    def filter_test_details(self):
        test_details = 'test_details.job_name:{} '.format(
            self.test_doc['_source']['test_details']['job_name'].split('/')[0])
        test_details += self.test_cmd_details()
        test_details += ' AND test_details.time_completed: {}'.format(self.date_re)
        return test_details

    def test_cmd_details(self):
        raise NotImplementedError('Derived classes must implement this method.')

    def __call__(self, *args, **kwargs):
        try:
            return '{} AND {}'.format(self.filter_test_details(), self.filter_setup_details())
        except KeyError:
            log.exception('Expected parameters for filtering are not found , test {} {}'.format(
                self.test_type, self.test_doc['_id']))
        return None


class QueryFilterCS(QueryFilter):

    _CMD = ('cassandra-stress', )
    _PRELOAD_CMD = ('preload-cassandra-stress', )
    _PARAMS = ('command', 'cl', 'rate threads', 'schema', 'mode', 'pop', 'duration')
    _PROFILE_PARAMS = ('command', 'profile', 'ops', 'rate threads', 'duration')

    def test_details_params(self):
        return self._CMD + self._PRELOAD_CMD if \
            self.test_doc['_source']['test_details'].get(self._PRELOAD_CMD[0]) else self._CMD

    def cs_params(self):
        return self._PROFILE_PARAMS if self.test_type.endswith('profiles') else self._PARAMS

    def test_cmd_details(self):
        test_details = ""
        for cs in self.test_details_params():
            for param in self.cs_params():
                if param == 'rate threads':
                    test_details += ' AND test_details.{}.rate\ threads: {}'.format(
                        cs, self.test_doc['_source']['test_details'][cs][param])
                elif param == 'duration' and cs.startswith('preload'):
                    continue
                else:
                    param_val = self.test_doc['_source']['test_details'][cs][param]
                    if param in ['profile', 'ops']:
                        param_val = "\"{}\"".format(param_val)
                    test_details += ' AND test_details.{}.{}: {}'.format(cs, param, param_val)
        return test_details


class QueryFilterScyllaBench(QueryFilter):

    _CMD = ('scylla-bench', )
{'fraction_non_alphanumeric': 0.12710654098828905, 'fraction_numerical': 0.0022850614110254213, 'mean_word_length': 2.8288682340076545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7451677', 'n_tokens_mistral': 2124, 'n_tokens_neox': 2051, 'n_words': 446}