<commit_before># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from __future__ import division

import re
import time
from collections import defaultdict
from copy import deepcopy

import redis
from six import iteritems

from datadog_checks.base import AgentCheck, ensure_unicode, is_affirmative
from datadog_checks.base.utils.common import round_value

DEFAULT_MAX_SLOW_ENTRIES = 128
MAX_SLOW_ENTRIES_KEY = "slowlog-max-len"

REPL_KEY = 'master_link_status'
LINK_DOWN_KEY = 'master_link_down_since_seconds'


class Redis(AgentCheck):
    db_key_pattern = re.compile(r'^db\d+')
    slave_key_pattern = re.compile(r'^slave\d+')
    subkeys = ['keys', 'expires']

    SOURCE_TYPE_NAME = 'redis'

    GAUGE_KEYS = {
        # Append-only metrics
        'aof_last_rewrite_time_sec': 'redis.aof.last_rewrite_time',
        'aof_rewrite_in_progress': 'redis.aof.rewrite',
        'aof_current_size': 'redis.aof.size',
        'aof_buffer_length': 'redis.aof.buffer_length',
        # Network
        'connected_clients': 'redis.net.clients',
        'connected_slaves': 'redis.net.slaves',
        'rejected_connections': 'redis.net.rejected',
        # clients
        'blocked_clients': 'redis.clients.blocked',
        'client_biggest_input_buf': 'redis.clients.biggest_input_buf',
        'client_longest_output_list': 'redis.clients.longest_output_list',
        # Keys
        'evicted_keys': 'redis.keys.evicted',
        'expired_keys': 'redis.keys.expired',
        # stats
        'latest_fork_usec': 'redis.perf.latest_fork_usec',
        'bytes_received_per_sec': 'redis.bytes_received_per_sec',
        'bytes_sent_per_sec': 'redis.bytes_sent_per_sec',
        # Note: 'bytes_received_per_sec' and 'bytes_sent_per_sec' are only
        # available on Azure Redis
        # pubsub
        'pubsub_channels': 'redis.pubsub.channels',
        'pubsub_patterns': 'redis.pubsub.patterns',
        # rdb
        'rdb_bgsave_in_progress': 'redis.rdb.bgsave',
        'rdb_changes_since_last_save': 'redis.rdb.changes_since_last',
        'rdb_last_bgsave_time_sec': 'redis.rdb.last_bgsave_time',
        # memory
        'mem_fragmentation_ratio': 'redis.mem.fragmentation_ratio',
        'used_memory': 'redis.mem.used',
        'used_memory_lua': 'redis.mem.lua',
        'used_memory_peak': 'redis.mem.peak',
        'used_memory_rss': 'redis.mem.rss',
        'maxmemory': 'redis.mem.maxmemory',
        # replication
        'master_last_io_seconds_ago': 'redis.replication.last_io_seconds_ago',
        'master_sync_in_progress': 'redis.replication.sync',
        'master_sync_left_bytes': 'redis.replication.sync_left_bytes',
        'repl_backlog_histlen': 'redis.replication.backlog_histlen',
        'master_repl_offset': 'redis.replication.master_repl_offset',
        'slave_repl_offset': 'redis.replication.slave_repl_offset',
    }

    RATE_KEYS = {
        # cpu
        'used_cpu_sys': 'redis.cpu.sys',
        'used_cpu_sys_children': 'redis.cpu.sys_children',
        'used_cpu_user': 'redis.cpu.user',
        'used_cpu_user_children': 'redis.cpu.user_children',
        # stats
        'keyspace_hits': 'redis.stats.keyspace_hits',
        'keyspace_misses': 'redis.stats.keyspace_misses',
    }

    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self.connections = {}
        self.last_timestamp_seen = defaultdict(int)

    def get_library_versions(self):
        return {"redis": redis.__version__}

    def _parse_dict_string(self, string, key, default):
        """Take from a more recent redis.py, parse_info"""
        try:
            for item in string.split(','):
                k, v = item.rsplit('=', 1)
                if k == key:
                    try:
                        return int(v)
                    except ValueError:
                        return v
            return default
        except Exception:
            self.log.exception("Cannot parse dictionary string: %s" % string)
            return default

    def _generate_instance_key(self, instance):
        if 'unix_socket_path' in instance:
            return instance.get('unix_socket_path'), instance.get('db')
        else:
            return instance.get('host'), instance.get('port'), instance.get('db')

    def _get_conn(self, instance):
        no_cache = is_affirmative(instance.get('disable_connection_cache', False))
        key = self._generate_instance_key(instance)

        if no_cache or key not in self.connections:
            try:
                # Only send useful parameters to the redis client constructor
                list_params = [
                    'host',
                    'port',
                    'db',
                    'password',
                    'socket_timeout',
                    'connection_pool',
                    'charset',
                    'errors',
                    'unix_socket_path',
                    'ssl',
                    'ssl_certfile',
                    'ssl_keyfile',
                    'ssl_ca_certs',
                    'ssl_cert_reqs',
                ]

                # Set a default timeout (in seconds) if no timeout is specified in the instance config
                instance['socket_timeout'] = instance.get('socket_timeout', 5)
                connection_params = dict((k, instance[k]) for k in list_params if k in instance)
                # If caching is disabled, we overwrite the dictionary value so the old connection
                # will be closed as soon as the corresponding Python object gets garbage collected
                self.connections[key] = redis.Redis(**connection_params)

            except TypeError:
                msg = "You need a redis library that supports authenticated connections. Try `pip install redis`."
                raise Exception(msg)

        return self.connections[key]

    def _get_tags(self, custom_tags, instance):
        tags = set(custom_tags or [])

        if 'unix_socket_path' in instance:
            tags_to_add = ["redis_host:%s" % instance.get("unix_socket_path"), "redis_port:unix_socket"]
        else:
            tags_to_add = ["redis_host:%s" % instance.get('host'), "redis_port:%s" % instance.get('port')]

        tags = sorted(tags.union(tags_to_add))

        return tags

    def _check_db(self, instance, custom_tags=None):
        conn = self._get_conn(instance)
        tags = self._get_tags(custom_tags, instance)

        # Ping the database for info, and track the latency.
        # Process the service check: the check passes if we can connect to Redis
        start = time.time()
        try:
            info = conn.info()
            tags = sorted(tags + ["redis_role:%s" % info["role"]])
            status = AgentCheck.OK
            self.service_check('redis.can_connect', status, tags=tags)
            self._collect_metadata(info)
        except ValueError:
            status = AgentCheck.CRITICAL
            self.service_check('redis.can_connect', status, tags=tags)
            raise
        except Exception:
            status = AgentCheck.CRITICAL
            self.service_check('redis.can_connect', status, tags=tags)
            raise

        latency_ms = round_value((time.time() - start) * 1000, 2)
        self.gauge('redis.info.latency_ms', latency_ms, tags=tags)

        # Save the database statistics.
        for key in info.keys():
            if self.db_key_pattern.match(key):
                db_tags = tags + ["redis_db:" + key]
                # allows tracking percentage of expired keys as DD does not
                # currently allow arithmetic on metric for monitoring
                expires_keys = info[key]["expires"]
                total_keys = info[key]["keys"]
                persist_keys = total_keys - expires_keys
                self.gauge("redis.persist", persist_keys, tags=db_tags)
                self.gauge("redis.persist.percent", 100 * persist_keys / total_keys, tags=db_tags)
                self.gauge("redis.expires.percent", 100 * expires_keys / total_keys, tags=db_tags)<commit_msg>Adjust latency tracking in redisdb integration (#3689)<commit_after># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from __future__ import division

import re
import time
from collections import defaultdict
from copy import deepcopy

import redis
from six import iteritems

from datadog_checks.base import AgentCheck, ensure_unicode, is_affirmative
from datadog_checks.base.utils.common import round_value

DEFAULT_MAX_SLOW_ENTRIES = 128
MAX_SLOW_ENTRIES_KEY = "slowlog-max-len"

REPL_KEY = 'master_link_status'
LINK_DOWN_KEY = 'master_link_down_since_seconds'


class Redis(AgentCheck):
    db_key_pattern = re.compile(r'^db\d+')
    slave_key_pattern = re.compile(r'^slave\d+')
    subkeys = ['keys', 'expires']

    SOURCE_TYPE_NAME = 'redis'

    GAUGE_KEYS = {
        # Append-only metrics
        'aof_last_rewrite_time_sec': 'redis.aof.last_rewrite_time',
        'aof_rewrite_in_progress': 'redis.aof.rewrite',
        'aof_current_size': 'redis.aof.size',
        'aof_buffer_length': 'redis.aof.buffer_length',
        # Network
        'connected_clients': 'redis.net.clients',
        'connected_slaves': 'redis.net.slaves',
        'rejected_connections': 'redis.net.rejected',
        # clients
        'blocked_clients': 'redis.clients.blocked',
        'client_biggest_input_buf': 'redis.clients.biggest_input_buf',
        'client_longest_output_list': 'redis.clients.longest_output_list',
        # Keys
        'evicted_keys': 'redis.keys.evicted',
        'expired_keys': 'redis.keys.expired',
        # stats
        'latest_fork_usec': 'redis.perf.latest_fork_usec',
        'bytes_received_per_sec': 'redis.bytes_received_per_sec',
        'bytes_sent_per_sec': 'redis.bytes_sent_per_sec',
        # Note: 'bytes_received_per_sec' and 'bytes_sent_per_sec' are only
        # available on Azure Redis
        # pubsub
        'pubsub_channels': 'redis.pubsub.channels',
        'pubsub_patterns': 'redis.pubsub.patterns',
        # rdb
        'rdb_bgsave_in_progress': 'redis.rdb.bgsave',
        'rdb_changes_since_last_save': 'redis.rdb.changes_since_last',
        'rdb_last_bgsave_time_sec': 'redis.rdb.last_bgsave_time',
        # memory
        'mem_fragmentation_ratio': 'redis.mem.fragmentation_ratio',
        'used_memory': 'redis.mem.used',
        'used_memory_lua': 'redis.mem.lua',
        'used_memory_peak': 'redis.mem.peak',
        'used_memory_rss': 'redis.mem.rss',
        'maxmemory': 'redis.mem.maxmemory',
        # replication
        'master_last_io_seconds_ago': 'redis.replication.last_io_seconds_ago',
        'master_sync_in_progress': 'redis.replication.sync',
        'master_sync_left_bytes': 'redis.replication.sync_left_bytes',
        'repl_backlog_histlen': 'redis.replication.backlog_histlen',
        'master_repl_offset': 'redis.replication.master_repl_offset',
        'slave_repl_offset': 'redis.replication.slave_repl_offset',
    }

    RATE_KEYS = {
        # cpu
        'used_cpu_sys': 'redis.cpu.sys',
        'used_cpu_sys_children': 'redis.cpu.sys_children',
        'used_cpu_user': 'redis.cpu.user',
        'used_cpu_user_children': 'redis.cpu.user_children',
        # stats
        'keyspace_hits': 'redis.stats.keyspace_hits',
        'keyspace_misses': 'redis.stats.keyspace_misses',
    }

    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self.connections = {}
        self.last_timestamp_seen = defaultdict(int)

    def get_library_versions(self):
        return {"redis": redis.__version__}

    def _parse_dict_string(self, string, key, default):
        """Take from a more recent redis.py, parse_info"""
        try:
            for item in string.split(','):
                k, v = item.rsplit('=', 1)
                if k == key:
                    try:
                        return int(v)
                    except ValueError:
                        return v
            return default
        except Exception:
            self.log.exception("Cannot parse dictionary string: %s" % string)
            return default

    def _generate_instance_key(self, instance):
        if 'unix_socket_path' in instance:
            return instance.get('unix_socket_path'), instance.get('db')
        else:
            return instance.get('host'), instance.get('port'), instance.get('db')

    def _get_conn(self, instance):
        no_cache = is_affirmative(instance.get('disable_connection_cache', False))
        key = self._generate_instance_key(instance)

        if no_cache or key not in self.connections:
            try:
                # Only send useful parameters to the redis client constructor
                list_params = [
                    'host',
                    'port',
                    'db',
                    'password',
                    'socket_timeout',
                    'connection_pool',
                    'charset',
                    'errors',
                    'unix_socket_path',
                    'ssl',
                    'ssl_certfile',
                    'ssl_keyfile',
                    'ssl_ca_certs',
                    'ssl_cert_reqs',
                ]

                # Set a default timeout (in seconds) if no timeout is specified in the instance config
                instance['socket_timeout'] = instance.get('socket_timeout', 5)
                connection_params = dict((k, instance[k]) for k in list_params if k in instance)
                # If caching is disabled, we overwrite the dictionary value so the old connection
                # will be closed as soon as the corresponding Python object gets garbage collected
                self.connections[key] = redis.Redis(**connection_params)

            except TypeError:
                msg = "You need a redis library that supports authenticated connections. Try `pip install redis`."
                raise Exception(msg)

        return self.connections[key]

    def _get_tags(self, custom_tags, instance):
        tags = set(custom_tags or [])

        if 'unix_socket_path' in instance:
            tags_to_add = ["redis_host:%s" % instance.get("unix_socket_path"), "redis_port:unix_socket"]
        else:
            tags_to_add = ["redis_host:%s" % instance.get('host'), "redis_port:%s" % instance.get('port')]

        tags = sorted(tags.union(tags_to_add))

        return tags

    def _check_db(self, instance, custom_tags=None):
        conn = self._get_conn(instance)
        tags = self._get_tags(custom_tags, instance)

        # Ping the database for info, and track the latency.
        # Process the service check: the check passes if we can connect to Redis
        start = time.time()
        try:
            info = conn.info()
            latency_ms = round_value((time.time() - start) * 1000, 2)
            tags = sorted(tags + ["redis_role:%s" % info["role"]])
            self.gauge('redis.info.latency_ms', latency_ms, tags=tags)
            status = AgentCheck.OK
            self.service_check('redis.can_connect', status, tags=tags)
            self._collect_metadata(info)
        except ValueError:
            status = AgentCheck.CRITICAL
            self.service_check('redis.can_connect', status, tags=tags)
            raise
        except Exception:
            status = AgentCheck.CRITICAL
            self.service_check('redis.can_connect', status, tags=tags)
            raise

        # Save the database statistics.
        for key in info.keys():
            if self.db_key_pattern.match(key):
                db_tags = tags + ["redis_db:" + key]
                # allows tracking percentage of expired keys as DD does not
                # currently allow arithmetic on metric for monitoring
                expires_keys = info[key]["expires"]
                total_keys = info[key]["keys"]
                persist_keys = total_keys - expires_keys
                self.gauge("redis.persist", persist_keys, tags=db_tags)
                self.gauge("redis.persist.percent", 100 * persist_keys / total_keys, tags=db_tags)
                self.gauge("redis.expires.percent", 100 * expires_keys / total_keys, tags=db_tags)
{'fraction_non_alphanumeric': 0.10396641927241757, 'fraction_numerical': 0.0027983939652025795, 'mean_word_length': 2.331779489258208, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6259154', 'n_tokens_mistral': 5103, 'n_tokens_neox': 4849, 'n_words': 1021}