<commit_before>"""
Originally from https://github.com/lesaux/diamond-DockerContainerCollector
"""

import docker
import diamond.collector
import traceback
try:
    import json
except ImportError:
    import simplejson as json
from diamond.collector import str_to_bool


class NetuitiveDockerCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(
            NetuitiveDockerCollector, self).get_default_config_help()
        config_help.update({
            'simple': 'Only collect total metrics for CPU, Memory',
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(NetuitiveDockerCollector, self).get_default_config()
        config.update({
            'path':     'containers',
            'simple':   'False'
        })
        return config

    def flatten_dict(self, d):
        def items():
            for key, value in d.items():
                if isinstance(value, dict):
                    for subkey, subvalue in self.flatten_dict(value).items():
                        yield key + "." + subkey, subvalue
                else:
                    yield key, value
        return dict(items())

    def collect(self):

        def filter_metric(metric):
            if not str_to_bool(self.config['simple']):
                return True
            else:
                if metric.startswith('memory.stats.'):
                    if not metric.startswith('total', 13):
                        return False
            return True

        def print_metric(cc, name):
            data = cc.stats(name)
            metrics = json.loads(data.next())
            # memory metrics
            self.memory = self.flatten_dict(metrics['memory_stats'])
            for key, value in self.memory.items():
                if value is not None:
                    metric = 'memory.' + key
                    if filter_metric(metric):
                        metric_name = name + "." + metric
                        self.publish_gauge(metric_name, value)
            # cpu metrics
            self.cpu = self.flatten_dict(metrics['cpu_stats'])
            for key, value in self.cpu.items():
                if value is not None:
                    # percpu_usage is a list, we'll deal with it after
                    if type(value) == int:
                        metric_name = name + ".cpu." + key
                        self.publish_counter(metric_name, value)
                    # dealing with percpu_usage
                    if type(value) == list and not str_to_bool(self.config['simple']):
                        self.length = len(value)
                        for i in range(self.length):
                            self.value = value
                            self.metric_name = name + ".cpu." + key + str(i)
                            self.publish_counter(
                                self.metric_name, self.value[i])
            
            # network metrics
            self.network = None

            if 'network' in metrics:
                self.network = self.flatten_dict(metrics['network'])

            if 'networks' in metrics:
                self.network = self.flatten_dict(metrics['networks'])

            if self.network is not None:
                for key, value in self.network.items():
                    if value is not None:
                        metric_name = name + ".network." + key
                        self.publish_counter(metric_name, value)
            
            # blkio metrics
            self.blkio = self.flatten_dict(metrics['blkio_stats'])
            for key, value in self.blkio.items():
                if value is not None and not isinstance(value, list):
                    metric_name = name + ".blkio." + key
                    self.publish_counter(metric_name, value)

        cc = docker.Client(
            base_url='unix://var/run/docker.sock', version='auto')
        dockernames = [i['Names'] for i in cc.containers()]

        running_containers = len(cc.containers())
        all_containers = len(cc.containers(all=True))
        stopped_containers = (all_containers - running_containers)

        image_count = len(set(cc.images(quiet=True)))
        dangling_image_count = len(
            set(cc.images(quiet=True, all=True, filters={'dangling': True})))

        self.publish('counts.running', running_containers)
        self.publish('counts.stopped', stopped_containers)

        self.publish('counts.all_containers', all_containers)
        self.publish('counts.images', image_count)
        self.publish('counts.dangling_images', dangling_image_count)

        for dname in dockernames:
            name = next(n for n in dname if n.count('/') == 1)
            try:
                print_metric(cc, name[1:])
            except Exception as e:
                self.log.error('Unable to collect for container ' +
                               name[1:] + ': ' + traceback.format_exc())<commit_msg>Add a Docker collector minimal mode<commit_after>"""
Originally from https://github.com/lesaux/diamond-DockerContainerCollector
"""

import docker
import diamond.collector
import traceback
try:
    import json
except ImportError:
    import simplejson as json
from diamond.collector import str_to_bool


class NetuitiveDockerCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(
            NetuitiveDockerCollector, self).get_default_config_help()
        config_help.update({
            'simple': 'Only collect total metrics for CPU, Memory',
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(NetuitiveDockerCollector, self).get_default_config()
        config.update({
            'path':     'containers',
            'simple':   'False',
            'minimal':   'False'
        })
        return config

    def flatten_dict(self, d):
        def items():
            for key, value in d.items():
                if isinstance(value, dict):
                    for subkey, subvalue in self.flatten_dict(value).items():
                        yield key + "." + subkey, subvalue
                else:
                    yield key, value
        return dict(items())

    def collect(self):

        def filter_metric(metric):
            if not str_to_bool(self.config['simple']):
                return True
            else:
                if metric.startswith('memory.stats.'):
                    if not metric.startswith('total', 13):
                        return False
            return True

        def print_metric(cc, name):
            data = cc.stats(name)
            metrics = json.loads(data.next())
            # memory metrics
            self.memory = self.flatten_dict(metrics['memory_stats'])
            for key, value in self.memory.items():
                if value is not None:
                    metric = 'memory.' + key
                    if filter_metric(metric):
                        metric_name = name + "." + metric
                        self.publish_gauge(metric_name, value)
            # cpu metrics
            self.cpu = self.flatten_dict(metrics['cpu_stats'])
            for key, value in self.cpu.items():
                if value is not None:
                    # percpu_usage is a list, we'll deal with it after
                    if type(value) == int:
                        metric_name = name + ".cpu." + key
                        self.publish_counter(metric_name, value)
                    # dealing with percpu_usage
                    if type(value) == list and not str_to_bool(self.config['simple']):
                        self.length = len(value)
                        for i in range(self.length):
                            self.value = value
                            self.metric_name = name + ".cpu." + key + str(i)
                            self.publish_counter(
                                self.metric_name, self.value[i])
            
            # network metrics
            self.network = None

            if 'network' in metrics:
                self.network = self.flatten_dict(metrics['network'])

            if 'networks' in metrics:
                self.network = self.flatten_dict(metrics['networks'])

            if self.network is not None:
                for key, value in self.network.items():
                    if value is not None:
                        metric_name = name + ".network." + key
                        self.publish_counter(metric_name, value)
            
            # blkio metrics
            self.blkio = self.flatten_dict(metrics['blkio_stats'])
            for key, value in self.blkio.items():
                if value is not None and not isinstance(value, list):
                    metric_name = name + ".blkio." + key
                    self.publish_counter(metric_name, value)

        def print_minimal_metric(cc, name):
            data = cc.stats(name)
            metrics = json.loads(data.next())
            # memory metrics
            self.memory = self.flatten_dict(metrics['memory_stats'])

            self.publish(name + '.netuitive.docker.memory.container_memory_percent', 100.0 * self.memory['usage'] / self.memory['limit'])

            # cpu metrics
            self.cpu = self.flatten_dict(metrics['cpu_stats'])

            usage = self.derivative('cpu.cpu_usage.total_usage', self.cpu['cpu_usage.total_usage'], diamond.collector.MAX_COUNTER)
            total = self.derivative('cpu.system_cpu_usage', self.cpu['system_cpu_usage'], diamond.collector.MAX_COUNTER)

            # Derivatives take one cycle to warm up
            if total != 0:
                self.publish(name + '.netuitive.docker.cpu.container_cpu_percent', 100.0 * usage / total)

        cc = docker.Client(
            base_url='unix://var/run/docker.sock', version='auto')
        dockernames = [i['Names'] for i in cc.containers()]

        running_containers = len(cc.containers())
        all_containers = len(cc.containers(all=True))
        stopped_containers = (all_containers - running_containers)

        image_count = len(set(cc.images(quiet=True)))
        dangling_image_count = len(
            set(cc.images(quiet=True, all=True, filters={'dangling': True})))

        self.publish('counts.running', running_containers)
        self.publish('counts.stopped', stopped_containers)

        self.publish('counts.all_containers', all_containers)
        self.publish('counts.images', image_count)
        self.publish('counts.dangling_images', dangling_image_count)

        for dname in dockernames:
            name = next(n for n in dname if n.count('/') == 1)
            try:
                print_minimal_metric(cc, name[1:]) if str_to_bool(self.config['minimal']) else print_metric(cc, name[1:])
            except Exception as e:
                self.log.error('Unable to collect for container ' +
                               name[1:] + ': ' + traceback.format_exc())
{'fraction_non_alphanumeric': 0.1003584229390681, 'fraction_numerical': 0.0017921146953405018, 'mean_word_length': 1.680999279365842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4146738', 'n_tokens_mistral': 3018, 'n_tokens_neox': 2846, 'n_words': 690}