<commit_before>from cbagent.collectors import Collector

class SecondaryStats(Collector):

    COLLECTOR = "secondary_stats"

    def _get_secondary_stats(self, bucket):
        server = self.master_node
        server = server.split(':')[0]
        uri = "/pools/default/buckets/@index-{}/stats".format(bucket)
        samples = self.get_http(path=uri, server=server)
        stats = dict()
        for metric, values in samples['op']['samples'].iteritems():
            metric = metric.replace('/', '_')
            stats[metric] = values[-1]  # only the most recent sample
        return stats

    def sample(self):
        for bucket in self.get_buckets():
            stats = self._get_secondary_stats(bucket)
            if stats:
               self.update_metric_metadata(stats.keys())
               self.store.append(stats,
                                 cluster=self.cluster,
                                 collector=self.COLLECTOR)

    def update_metadata(self):
        self.mc.add_cluster()

        for bucket in self.get_buckets():
            self.mc.add_bucket(bucket)
<commit_msg>add bucket information to 2i stats sample<commit_after>from cbagent.collectors import Collector

class SecondaryStats(Collector):

    COLLECTOR = "secondary_stats"

    def _get_secondary_stats(self, bucket):
        server = self.master_node
        server = server.split(':')[0]
        uri = "/pools/default/buckets/@index-{}/stats".format(bucket)
        samples = self.get_http(path=uri, server=server)
        stats = dict()
        for metric, values in samples['op']['samples'].iteritems():
            metric = metric.replace('/', '_')
            stats[metric] = values[-1]  # only the most recent sample
        return stats

    def sample(self):
        for bucket in self.get_buckets():
            stats = self._get_secondary_stats(bucket)
            if stats:
               self.update_metric_metadata(stats.keys())
               self.store.append(stats,cluster=self.cluster,
                                 bucket=bucket, collector=self.COLLECTOR)

    def update_metadata(self):
        self.mc.add_cluster()

        for bucket in self.get_buckets():
            self.mc.add_bucket(bucket)

{'fraction_non_alphanumeric': 0.1117620549797206, 'fraction_numerical': 0.002253267237494367, 'mean_word_length': 2.217391304347826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5270020', 'n_tokens_mistral': 622, 'n_tokens_neox': 597, 'n_words': 126}