<commit_before># (C) Datadog, Inc. 2010-2016
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

# std
import logging
import simplejson as json

# 3rd party
from docker.errors import NullResource, NotFound

# project
from utils.dockerutil import DockerUtil
from utils.kubernetes import KubeUtil
from utils.platform import Platform
from utils.service_discovery.abstract_sd_backend import AbstractSDBackend
from utils.service_discovery.config_stores import get_config_store

DATADOG_ID = 'com.datadoghq.sd.check.id'
log = logging.getLogger(__name__)


class _SDDockerBackendConfigFetchState(object):
    def __init__(self, inspect_fn, kube_pods=None):
        self.inspect_cache = {}

        self.inspect_fn = inspect_fn
        self.kube_pods = kube_pods

    def inspect_container(self, c_id):
        if c_id in self.inspect_cache:
            return self.inspect_cache[c_id]

        try:
            self.inspect_cache[c_id] = inspect = self.inspect_fn(c_id)
        except (NullResource, NotFound):
            self.inspect_cache[c_id] = inspect = {}

        return inspect

    def get_kube_container_status(self, c_id):
        for pod in self.kube_pods:
            c_statuses = pod.get('status', {}).get('containerStatuses', [])
            for status in c_statuses:
                if c_id == status.get('containerID', '').split('//')[-1]:
                    return status
        return {}

    def get_kube_container_name(self, c_id):
        return self.get_kube_container_status(c_id).get('name')

    def get_kube_container_spec(self, c_id):
        c_name = self.get_kube_container_name(c_id)
        containers = self.get_kube_config(c_id, 'spec').get('containers', [])
        for co in containers:
            if co.get('name') == c_name:
                return co
        return None

    def get_kube_config(self, c_id, key):
        """Get a part of a pod config from the kubernetes API"""
        for pod in self.kube_pods:
            c_statuses = pod.get('status', {}).get('containerStatuses', [])
            for status in c_statuses:
                if c_id == status.get('containerID', '').split('//')[-1]:
                    return pod.get(key, {})
        return {}


class SDDockerBackend(AbstractSDBackend):
    """Docker-based service discovery"""

    def __init__(self, agentConfig):
        try:
            self.config_store = get_config_store(agentConfig=agentConfig)
        except Exception as e:
            log.error('Failed to instantiate the config store client. '
                      'Auto-config only will be used. %s' % str(e))
            agentConfig['sd_config_backend'] = None
            self.config_store = get_config_store(agentConfig=agentConfig)

        self.docker_client = DockerUtil(config_store=self.config_store).client
        if Platform.is_k8s():
            self.kubeutil = KubeUtil()

        self.VAR_MAPPING = {
            'host': self._get_host_address,
            'port': self._get_port,
            'tags': self._get_additional_tags,
        }

        AbstractSDBackend.__init__(self, agentConfig)

    def _make_fetch_state(self):
        return _SDDockerBackendConfigFetchState(
            self.docker_client.inspect_container,
            self.kubeutil.retrieve_pods_list().get('items', []) if Platform.is_k8s() else None)

    def update_checks(self, changed_containers):
        state = self._make_fetch_state()

        conf_reload_set = set()
        for c_id in changed_containers:
            checks = self._get_checks_to_refresh(state, c_id)
            conf_reload_set.update(set(checks))

        if conf_reload_set:
            self.reload_check_configs = conf_reload_set

    def _get_checks_to_refresh(self, state, c_id):
        """Get the list of checks applied to a container from the identifier_to_checks cache in the config store.
        Use the DATADOG_ID label or the image."""
        inspect = state.inspect_container(c_id)
        identifier = inspect.get('Config', {}).get('Labels', {}).get(DATADOG_ID) or \
            inspect.get('Config', {}).get('Image')

        platform_kwargs = {}
        if Platform.is_k8s():
            kube_metadata = state.get_kube_config(c_id, 'metadata') or {}
            platform_kwargs = {
                'kube_annotations': kube_metadata.get('annotations'),
                'kube_container_name': state.get_kube_container_name(c_id),
            }

        return self.config_store.get_checks_to_refresh(identifier, **platform_kwargs)

    def _get_host_address(self, state, c_id, tpl_var):
        """Extract the container IP from a docker inspect object, or the kubelet API."""
        c_inspect = state.inspect_container(c_id)
        c_id, c_img = c_inspect.get('Id', ''), c_inspect.get('Config', {}).get('Image', '')

        networks = c_inspect.get('NetworkSettings', {}).get('Networks') or {}
        ip_dict = {}
        for net_name, net_desc in networks.iteritems():
            ip = net_desc.get('IPAddress')
            if ip:
                ip_dict[net_name] = ip
        ip_addr = self._extract_ip_from_networks(ip_dict, tpl_var)<commit_msg>[service discovery] properly handle config reload for removed containers<commit_after># (C) Datadog, Inc. 2010-2016
# All rights reserved
# Licensed under Simplified BSD License (see LICENSE)

# std
import logging
import simplejson as json

# 3rd party
from docker.errors import NullResource, NotFound

# project
from utils.dockerutil import DockerUtil
from utils.kubernetes import KubeUtil
from utils.platform import Platform
from utils.service_discovery.abstract_sd_backend import AbstractSDBackend
from utils.service_discovery.config_stores import get_config_store

DATADOG_ID = 'com.datadoghq.sd.check.id'
log = logging.getLogger(__name__)


class _SDDockerBackendConfigFetchState(object):
    def __init__(self, inspect_fn, kube_pods=None):
        self.inspect_cache = {}

        self.inspect_fn = inspect_fn
        self.kube_pods = kube_pods

    def inspect_container(self, c_id):
        if c_id in self.inspect_cache:
            return self.inspect_cache[c_id]

        try:
            self.inspect_cache[c_id] = inspect = self.inspect_fn(c_id)
        except (NullResource, NotFound):
            self.inspect_cache[c_id] = inspect = {}

        return inspect

    def get_kube_container_status(self, c_id):
        for pod in self.kube_pods:
            c_statuses = pod.get('status', {}).get('containerStatuses', [])
            for status in c_statuses:
                if c_id == status.get('containerID', '').split('//')[-1]:
                    return status
        return {}

    def get_kube_container_name(self, c_id):
        return self.get_kube_container_status(c_id).get('name')

    def get_kube_container_spec(self, c_id):
        c_name = self.get_kube_container_name(c_id)
        containers = self.get_kube_config(c_id, 'spec').get('containers', [])
        for co in containers:
            if co.get('name') == c_name:
                return co
        return None

    def get_kube_config(self, c_id, key):
        """Get a part of a pod config from the kubernetes API"""
        for pod in self.kube_pods:
            c_statuses = pod.get('status', {}).get('containerStatuses', [])
            for status in c_statuses:
                if c_id == status.get('containerID', '').split('//')[-1]:
                    return pod.get(key, {})
        return {}


class SDDockerBackend(AbstractSDBackend):
    """Docker-based service discovery"""

    def __init__(self, agentConfig):
        try:
            self.config_store = get_config_store(agentConfig=agentConfig)
        except Exception as e:
            log.error('Failed to instantiate the config store client. '
                      'Auto-config only will be used. %s' % str(e))
            agentConfig['sd_config_backend'] = None
            self.config_store = get_config_store(agentConfig=agentConfig)

        self.docker_client = DockerUtil(config_store=self.config_store).client
        if Platform.is_k8s():
            self.kubeutil = KubeUtil()

        self.VAR_MAPPING = {
            'host': self._get_host_address,
            'port': self._get_port,
            'tags': self._get_additional_tags,
        }

        AbstractSDBackend.__init__(self, agentConfig)

    def _make_fetch_state(self):
        return _SDDockerBackendConfigFetchState(
            self.docker_client.inspect_container,
            self.kubeutil.retrieve_pods_list().get('items', []) if Platform.is_k8s() else None)

    def update_checks(self, changed_containers):
        state = self._make_fetch_state()

        conf_reload_set = set()
        for c_id in changed_containers:
            checks = self._get_checks_to_refresh(state, c_id)
            conf_reload_set.update(set(checks))

        if conf_reload_set:
            self.reload_check_configs = conf_reload_set

    def _get_checks_to_refresh(self, state, c_id):
        """Get the list of checks applied to a container from the identifier_to_checks cache in the config store.
        Use the DATADOG_ID label or the image."""
        inspect = state.inspect_container(c_id)

        # if the container was removed we can't tell which check is concerned
        # so we have to reload everything
        if not inspect:
            self.reload_check_configs = True
            return

        identifier = inspect.get('Config', {}).get('Labels', {}).get(DATADOG_ID) or \
            inspect.get('Config', {}).get('Image')

        platform_kwargs = {}
        if Platform.is_k8s():
            kube_metadata = state.get_kube_config(c_id, 'metadata') or {}
            platform_kwargs = {
                'kube_annotations': kube_metadata.get('annotations'),
                'kube_container_name': state.get_kube_container_name(c_id),
            }

        return self.config_store.get_checks_to_refresh(identifier, **platform_kwargs)

    def _get_host_address(self, state, c_id, tpl_var):
        """Extract the container IP from a docker inspect object, or the kubelet API."""
        c_inspect = state.inspect_container(c_id)
        c_id, c_img = c_inspect.get('Id', ''), c_inspect.get('Config', {}).get('Image', '')

        networks = c_inspect.get('NetworkSettings', {}).get('Networks') or {}
        ip_dict = {}
        for net_name, net_desc in networks.iteritems():
            ip = net_desc.get('IPAddress')
            if ip:
                ip_dict[net_name] = ip
        ip_addr = self._extract_ip_from_networks(ip_dict, tpl_var)
{'fraction_non_alphanumeric': 0.10360231917118144, 'fraction_numerical': 0.0026613439787092482, 'mean_word_length': 2.642090688819661, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2002603', 'n_tokens_mistral': 3274, 'n_tokens_neox': 3166, 'n_words': 703}