<commit_before>import json
import time
from uuid import uuid4

from btrc import CouchbaseClient, StatsReporter
from couchbase import Couchbase
from logger import logger

from perfrunner.settings import SF_STORAGE


class BtrcReporter(object):

    def __init__(self, test):
        self.test = test

    def reset_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Resetting utilization stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            cb.reset_utilization_stats()

    def save_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving utilization stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('util_stats')

    def save_btree_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving B-tree stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('btree_stats')


class SFReporter(object):

    def __init__(self, test):
        self.test = test

    def _add_cluster(self):
        cluster = self.test.cluster_spec.name
        params = self.test.cluster_spec.get_parameters()
        try:
            cb = Couchbase.connect(bucket='clusters', **SF_STORAGE)
            cb.set(cluster, params)
        except Exception, e:
            logger.warn('Failed to add cluster, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}, {1}'.format(
                cluster, params))

    def _add_metric(self, metric, metric_info):
        if metric_info is None:
            metric_info = {
                'title': self.test.test_config.get_test_descr(),
                'cluster': self.test.cluster_spec.name,
                'larger_is_better': self.test.test_config.get_regression_criterion()
            }
        try:
            cb = Couchbase.connect(bucket='metrics', **SF_STORAGE)
            cb.set(metric, metric_info)
        except Exception, e:
            logger.warn('Failed to add cluster, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}, {1}'.format(metric,
                                                               metric_info))

    def _prepare_data(self, metric, value):
        key = uuid4().hex
        master_node = self.test.cluster_spec.get_masters().values()[0]
        build = self.test.rest.get_version(master_node)
        data = {'build': build, 'metric': metric, 'value': value}
        data.update(self.test.reports)
        return key, data

    def _mark_previous_as_obsolete(self, cb, benckmark):
        for row in cb.query('benchmarks', 'values_by_build_and_metric',
                            key=[benckmark['metric'], benckmark['build']]):
            doc = cb.get(row.docid)
            doc.value.update({'obsolete': True})
            cb.set(row.docid, doc.value)

    def _log_benchmark(self, metric, value):
        _, benckmark = self._prepare_data(metric, value)
        logger.info('Dry run stats: {0}'.format(benckmark))

    def _post_benckmark(self, metric, value):
        key, benckmark = self._prepare_data(metric, value)
        try:
            cb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self._mark_previous_as_obsolete(cb, benckmark)
            cb.set(key, benckmark)
        except Exception, e:
            logger.warn('Failed to post results, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}'.format(benckmark))

    def post_to_sf(self, value, metric=None, metric_info=None):
        if metric is None:
            metric = '{0}_{1}'.format(self.test.test_config.name,
                                      self.test.cluster_spec.name)

        stats_settings = self.test.test_config.get_stats_settings()

        if stats_settings.post_to_sf:
            self._add_metric(metric, metric_info)<commit_msg>log results in more pretty format<commit_after>import json
import time
from uuid import uuid4

from btrc import CouchbaseClient, StatsReporter
from couchbase import Couchbase
from logger import logger

from perfrunner.settings import SF_STORAGE


class BtrcReporter(object):

    def __init__(self, test):
        self.test = test

    def reset_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Resetting utilization stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            cb.reset_utilization_stats()

    def save_utilzation_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving utilization stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('util_stats')

    def save_btree_stats(self):
        for target in self.test.target_iterator:
            logger.info('Saving B-tree stats from {0}/{1}'.format(
                        target.node, target.bucket))
            cb = CouchbaseClient(target.node, target.bucket,
                                 target.username, target.password)
            reporter = StatsReporter(cb)
            reporter.report_stats('btree_stats')


class SFReporter(object):

    def __init__(self, test):
        self.test = test

    def _add_cluster(self):
        cluster = self.test.cluster_spec.name
        params = self.test.cluster_spec.get_parameters()
        try:
            cb = Couchbase.connect(bucket='clusters', **SF_STORAGE)
            cb.set(cluster, params)
        except Exception, e:
            logger.warn('Failed to add cluster, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}, {1}'.format(
                cluster, params))

    def _add_metric(self, metric, metric_info):
        if metric_info is None:
            metric_info = {
                'title': self.test.test_config.get_test_descr(),
                'cluster': self.test.cluster_spec.name,
                'larger_is_better': self.test.test_config.get_regression_criterion()
            }
        try:
            cb = Couchbase.connect(bucket='metrics', **SF_STORAGE)
            cb.set(metric, metric_info)
        except Exception, e:
            logger.warn('Failed to add cluster, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}, {1}'.format(metric,
                                                               metric_info))

    def _prepare_data(self, metric, value):
        key = uuid4().hex
        master_node = self.test.cluster_spec.get_masters().values()[0]
        build = self.test.rest.get_version(master_node)
        data = {'build': build, 'metric': metric, 'value': value}
        data.update(self.test.reports)
        return key, data

    def _mark_previous_as_obsolete(self, cb, benckmark):
        for row in cb.query('benchmarks', 'values_by_build_and_metric',
                            key=[benckmark['metric'], benckmark['build']]):
            doc = cb.get(row.docid)
            doc.value.update({'obsolete': True})
            cb.set(row.docid, doc.value)

    def _log_benchmark(self, metric, value):
        _, benckmark = self._prepare_data(metric, value)
        logger.info('Dry run stats: {0}'.format(
            json.dumps(benckmark, indent=4, sort_keys=True)
        ))

    def _post_benckmark(self, metric, value):
        key, benckmark = self._prepare_data(metric, value)
        try:
            cb = Couchbase.connect(bucket='benchmarks', **SF_STORAGE)
            self._mark_previous_as_obsolete(cb, benckmark)
            cb.set(key, benckmark)
        except Exception, e:
            logger.warn('Failed to post results, {0}'.format(e))
        else:
            logger.info('Successfully posted: {0}'.format(
                json.dumps(benckmark, indent=4, sort_keys=True)
            ))

    def post_to_sf(self, value, metric=None, metric_info=None):
        if metric is None:
            metric = '{0}_{1}'.format(self.test.test_config.name,
                                      self.test.cluster_spec.name)

        stats_settings = self.test.test_config.get_stats_settings()

        if stats_settings.post_to_sf:
            self._add_metric(metric, metric_info)
{'fraction_non_alphanumeric': 0.10779765849348354, 'fraction_numerical': 0.004638833664678595, 'mean_word_length': 2.088335607094134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6250220', 'n_tokens_mistral': 2619, 'n_tokens_neox': 2566, 'n_words': 524}