<commit_before>import re
from cbagent.collectors import Collector


class SecondaryStorageStats(Collector):

    COLLECTOR = "secondary_storage_stats"

    METRICS = "memory_size", "num_cached_pages", "num_pages", "num_pages_swapout", "num_pages_swapin", \
              "bytes_incoming", "bytes_written", "write_amp", "lss_fragmentation", "cache_hits", "cache_misses", \
              "cache_hit_ratio", "resident_ratio", "allocated", "freed", "reclaimed", "reclaim_pending", \
              "rcache_hits", "rcache_misses", "rcache_hit_ratio"

    def __init__(self, settings):
        super().__init__(settings)
        self.index_node = settings.index_node

    def get_all_indexes(self):
        for index in self.indexes:
            yield index, self.buckets[0]

    def _get_secondary_storage_stats(self):
        server = self.index_node
        port = '9102'
        uri = "/stats/storage"
        samples = self.get_http(path=uri, server=server, port=port, json=False)
        index_stats = dict()
        stats = dict()
        data = samples.split("\n")

        index = "unknown"
        store = "unknown"

        for line in data:
            if "Index Instance" in line:
                index_data = re.findall(":.*", line)
                index = index_data[0].split()[0][1:]
                index_stats[index] = dict()
                stats = dict()
                continue
            if "Store" in line:
                store = re.findall("[a-zA-Z]+", line)[0]
                continue
            data = line.split("=")
            if len(data) == 2:
                metric = data[0].strip()
                if metric in self.METRICS:
                    key = store + "_" + metric
                    stats[key] = float(data[1].strip().replace("%", ""))
            index_stats[index] = stats

        return index_stats
<commit_msg>Changes in storage/stats as per changes in storage stats for json formating<commit_after>from cbagent.collectors import Collector


class SecondaryStorageStats(Collector):

    COLLECTOR = "secondary_storage_stats"

    METRICS = "memory_size", "num_cached_pages", "num_pages", "num_pages_swapout", "num_pages_swapin", \
              "bytes_incoming", "bytes_written", "write_amp", "lss_fragmentation", "cache_hits", "cache_misses", \
              "cache_hit_ratio", "resident_ratio", "allocated", "freed", "reclaimed", "reclaim_pending", \
              "rcache_hits", "rcache_misses", "rcache_hit_ratio"

    def __init__(self, settings):
        super().__init__(settings)
        self.index_node = settings.index_node

    def get_all_indexes(self):
        for index in self.indexes:
            yield index, self.buckets[0]

    def _get_secondary_storage_stats(self):
        server = self.index_node
        port = '9102'
        uri = "/stats/storage"
        samples = self.get_http(path=uri, server=server, port=port)
        index_stats = dict()
        for sample in samples:
            stats = dict()
            index = sample["Index"].split(":")[1]

            for store in sample["Stats"]:
                for metric, value in sample["Stats"][store].items():
                    if metric in self.METRICS:
                        key = store + "_" + metric
                        stats[key] = value
            index_stats[index] = stats
        return index_stats

{'fraction_non_alphanumeric': 0.11965300628178283, 'fraction_numerical': 0.0053843852826802275, 'mean_word_length': 2.2092130518234163, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4431729', 'n_tokens_mistral': 1007, 'n_tokens_neox': 964, 'n_words': 215}