<commit_before>"""
This is the code that needs to be integrated into collectd when run in
production. It contains the python code that integrates into the python module
for collectd. It connects to one or more vCenter Servers and gathers the configured 
metrics from ESXi hosts and Virtual Machines.

The file is organized in multiple sections. The first section implements the
callback functions executed be collectd which is followed be a couple of helper
functions that separate out some code to make the rest more readable. The
helper classes section provides threads that are used to parallelize things and
make the plugin a lot faster.
"""
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import collectd
import logging
import threading
import time
import ssl
import re
from pysphere import VIServer

CONFIGS = []            # Stores the configuration as passed from collectd
ENVIRONMENT = {}        # Runtime data and object cache

#####################################################################################
# IMPLEMENTATION OF COLLECTD CALLBACK FUNCTIONS 
#####################################################################################

def configure_callback(conf):
    """Receive configuration block. This is called by collectd for every
    configuration block it finds for this module."""

    # Set some sensible default values
    name = None
    host = None
    port = 443 
    verbose = None
    username = 'root'                   
    password = '<PASSWORD>'                 
    host_counters = ['cpu.usage', 'mem.usage']
    vm_counters = ['cpu.usage','mem.usage']
    inventory_refresh_interval = 600  

    for node in conf.children:
        key = node.key.lower()
        val = node.values

        if key == 'name':
            name = val[0]
        elif key == 'host':
            host = val[0]
        elif key == 'port':
            port = int(val[0])    
        elif key == 'verbose':
            verbose = bool(val)
        elif key == 'username':
            username = val[0]
        elif key == 'password':
            password = val[0]
        elif key == 'host_counters':
            values = val[0].split(',')
            for m in values:
                if len(m) > 0:
                    host_counters.append(m.strip())
        elif key == 'vm_counters':
            values = val[0].split(',')
            for m in values:
                vm_counters.append(m.strip())
        elif key == 'inventory_refresh_interval':
            inventory_refresh_interval = val[0]
        else:
            collectd.warning('collectsphere plugin: Unknown config key: %s.' % key )
            continue

    collectd.info('Loaded config: name=%s, host=%s, port=%s, verbose=%s, username=%s, password=%s, host_metrics=%s, vm_metrics=%s, inventory_refresh_interval=%s' % (name, host, port, verbose, username, "******", len(host_counters), len(vm_counters), inventory_refresh_interval))

    CONFIGS.append({'name': name,'host': host,'port': port,'verbose': verbose,'username': username,'password': password, 'host_counters': host_counters,'vm_counters': vm_counters})

def init_callback():
    """ In this method we create environments for every configured vCenter
    Server. This includes creating the connection, reading in counter ID
    mapping tables and spawning inventory watch dogs. """

    # For every set of configuration received from collectd, a environment must
    # be created. Further, a inventory watch dog thread has to be spawned.
    for config in CONFIGS:
        env = create_environment(config)
      
        inventory = {}
        thread = InventoryWatchDog(env.get('conn'), env, 600)
        thread.start()

        env['inventory'] = inventory
        env['watchdog'] = thread

        # The environment is stored under the name of the config block
        ENVIRONMENT[config.get("name")] = env
 
def read_callback():
    """ This function is regularly executed by collectd. It is important to
    minimize the execution time of the function which is why a lot of caching
    is performed using the environment objects. """

    # We are going to spawn a lot of threads soon to speed up metric fetching.
    # References to the threads are stored here.
    threads = []
    
    # Walk through the existing environments
    for name in ENVIRONMENT.keys():
        env = ENVIRONMENT[name]
        collectd.info("Entering environment: " + name)
        conn = env['conn']

        # Reconnect to vCenter Server if necessary and give up if a connection
        # cannot be established.
        if(conn.is_connected() == False):
            collectd.info("Reconnecting")
            reconnect(name)
            conn = env['conn']
            if(conn.is_connected() == False):
                collectd.info("Reconnect unsuccessful. Giving up.")
                continue
        
        # fetch the instance of perf manager from the object cache
        pm = env['pm']

        # Host Metrics: Walk the _host_ inventory tree created by the inventory
        # watch dog and spawn a thread for every host.
        
        host_counter_ids = env.get('host_counter_ids')
        inventory = env.get('inventory')

        # Only do this if the inventory tree is not empty. If is isn't: for
        # every host in every cluster spawn a thread.
        if inventory:
            for cluster_name in inventory.keys():
                for host_name in inventory.get(cluster_name).get('hosts').keys():
                    
                    # fetch the host MOR from the inventory tree
                    host = inventory.get(cluster_name).get('hosts').get(host_name)
                   
                    # create thread and execute
                    thread = GetMetricsThread(pm, host, host_counter_ids, name, cluster_name, host_name.split('.')[0])
                    thread.start()
        
                    # append the thread to the list of threads
                    threads.append(thread)
        
        collectd.info("Spawned " + str(len(threads)) + " threads to fetch metrics.")
 
    # Wait for all threads to finish. Then dispatch all gathered values to collectd.
    for th in threads:
        th.join()

        # The thread was spawned with some information that we now need to
        # properly dispatch the values. Fetch that information now.
        vc_name = th.vc_name
        cluster_name = th.cluster_name
        host_name = th.host_name
        stats = th.stats

        # For every stat object work up the data, then dispatch
        for stat in stats:

            counter = stat.counter
            group = stat.group
            instance = stat.instance
            dtime = stat.time
            unit = stat.unit
            value = float(stat.value)
            timestamp = time.mktime(dtime.timetuple())

            # When the instance value is empty, the vSphere API references a
            # total. Example: A host has multiple cores for each of which we
            # get a single stat object. An additional stat object will be
            # returned by the vSphere API with an empty string for "instance".
            # This is the overall value accross all logical CPUs.
            if(len(stat.instance.strip()) == 0):
                instance = 'average'
           
            # We are limited to 63 characters for the type_instance field. This
            # is why we need to shorten NAA canonical names and VMFS UUIDs. For
            # NAAs it makes the most sense to use the last 6 characters while
            # UUIDs should be unique with the first 6. Unfortunately, there is
            # a chance to collisions for VMFS UUIDs.
            if 'naa' in instance:
                values = instance.split('.')
                instance = values[0] + values[1][-6:]

            if re.search('^[0-9a-f-]+$', instance):<commit_msg>renamed overall metrics from 'average' to 'all'<commit_after>"""
This is the code that needs to be integrated into collectd when run in
production. It contains the python code that integrates into the python module
for collectd. It connects to one or more vCenter Servers and gathers the configured 
metrics from ESXi hosts and Virtual Machines.

The file is organized in multiple sections. The first section implements the
callback functions executed be collectd which is followed be a couple of helper
functions that separate out some code to make the rest more readable. The
helper classes section provides threads that are used to parallelize things and
make the plugin a lot faster.
"""
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import collectd
import logging
import threading
import time
import ssl
import re
from pysphere import VIServer

CONFIGS = []            # Stores the configuration as passed from collectd
ENVIRONMENT = {}        # Runtime data and object cache

#####################################################################################
# IMPLEMENTATION OF COLLECTD CALLBACK FUNCTIONS 
#####################################################################################

def configure_callback(conf):
    """Receive configuration block. This is called by collectd for every
    configuration block it finds for this module."""

    # Set some sensible default values
    name = None
    host = None
    port = 443 
    verbose = None
    username = 'root'                   
    password = '<PASSWORD>'                 
    host_counters = ['cpu.usage', 'mem.usage']
    vm_counters = ['cpu.usage','mem.usage']
    inventory_refresh_interval = 600  

    for node in conf.children:
        key = node.key.lower()
        val = node.values

        if key == 'name':
            name = val[0]
        elif key == 'host':
            host = val[0]
        elif key == 'port':
            port = int(val[0])    
        elif key == 'verbose':
            verbose = bool(val)
        elif key == 'username':
            username = val[0]
        elif key == 'password':
            password = val[0]
        elif key == 'host_counters':
            values = val[0].split(',')
            for m in values:
                if len(m) > 0:
                    host_counters.append(m.strip())
        elif key == 'vm_counters':
            values = val[0].split(',')
            for m in values:
                vm_counters.append(m.strip())
        elif key == 'inventory_refresh_interval':
            inventory_refresh_interval = val[0]
        else:
            collectd.warning('collectsphere plugin: Unknown config key: %s.' % key )
            continue

    collectd.info('Loaded config: name=%s, host=%s, port=%s, verbose=%s, username=%s, password=%s, host_metrics=%s, vm_metrics=%s, inventory_refresh_interval=%s' % (name, host, port, verbose, username, "******", len(host_counters), len(vm_counters), inventory_refresh_interval))

    CONFIGS.append({'name': name,'host': host,'port': port,'verbose': verbose,'username': username,'password': password, 'host_counters': host_counters,'vm_counters': vm_counters})

def init_callback():
    """ In this method we create environments for every configured vCenter
    Server. This includes creating the connection, reading in counter ID
    mapping tables and spawning inventory watch dogs. """

    # For every set of configuration received from collectd, a environment must
    # be created. Further, a inventory watch dog thread has to be spawned.
    for config in CONFIGS:
        env = create_environment(config)
      
        inventory = {}
        thread = InventoryWatchDog(env.get('conn'), env, 600)
        thread.start()

        env['inventory'] = inventory
        env['watchdog'] = thread

        # The environment is stored under the name of the config block
        ENVIRONMENT[config.get("name")] = env
 
def read_callback():
    """ This function is regularly executed by collectd. It is important to
    minimize the execution time of the function which is why a lot of caching
    is performed using the environment objects. """

    # We are going to spawn a lot of threads soon to speed up metric fetching.
    # References to the threads are stored here.
    threads = []
    
    # Walk through the existing environments
    for name in ENVIRONMENT.keys():
        env = ENVIRONMENT[name]
        collectd.info("Entering environment: " + name)
        conn = env['conn']

        # Reconnect to vCenter Server if necessary and give up if a connection
        # cannot be established.
        if(conn.is_connected() == False):
            collectd.info("Reconnecting")
            reconnect(name)
            conn = env['conn']
            if(conn.is_connected() == False):
                collectd.info("Reconnect unsuccessful. Giving up.")
                continue
        
        # fetch the instance of perf manager from the object cache
        pm = env['pm']

        # Host Metrics: Walk the _host_ inventory tree created by the inventory
        # watch dog and spawn a thread for every host.
        
        host_counter_ids = env.get('host_counter_ids')
        inventory = env.get('inventory')

        # Only do this if the inventory tree is not empty. If is isn't: for
        # every host in every cluster spawn a thread.
        if inventory:
            for cluster_name in inventory.keys():
                for host_name in inventory.get(cluster_name).get('hosts').keys():
                    
                    # fetch the host MOR from the inventory tree
                    host = inventory.get(cluster_name).get('hosts').get(host_name)
                   
                    # create thread and execute
                    thread = GetMetricsThread(pm, host, host_counter_ids, name, cluster_name, host_name.split('.')[0])
                    thread.start()
        
                    # append the thread to the list of threads
                    threads.append(thread)
        
        collectd.info("Spawned " + str(len(threads)) + " threads to fetch metrics.")
 
    # Wait for all threads to finish. Then dispatch all gathered values to collectd.
    for th in threads:
        th.join()

        # The thread was spawned with some information that we now need to
        # properly dispatch the values. Fetch that information now.
        vc_name = th.vc_name
        cluster_name = th.cluster_name
        host_name = th.host_name
        stats = th.stats

        # For every stat object work up the data, then dispatch
        for stat in stats:

            counter = stat.counter
            group = stat.group
            instance = stat.instance
            dtime = stat.time
            unit = stat.unit
            value = float(stat.value)
            timestamp = time.mktime(dtime.timetuple())

            # When the instance value is empty, the vSphere API references a
            # total. Example: A host has multiple cores for each of which we
            # get a single stat object. An additional stat object will be
            # returned by the vSphere API with an empty string for "instance".
            # This is the overall value accross all logical CPUs.
            if(len(stat.instance.strip()) == 0):
                instance = 'all'
           
            # We are limited to 63 characters for the type_instance field. This
            # is why we need to shorten NAA canonical names and VMFS UUIDs. For
            # NAAs it makes the most sense to use the last 6 characters while
            # UUIDs should be unique with the first 6. Unfortunately, there is
            # a chance to collisions for VMFS UUIDs.
            if 'naa' in instance:
                values = instance.split('.')
                instance = values[0] + values[1][-6:]

            if re.search('^[0-9a-f-]+$', instance):
{'fraction_non_alphanumeric': 0.10241730279898219, 'fraction_numerical': 0.004071246819338422, 'mean_word_length': 2.2528450237947446, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670334', 'n_tokens_mistral': 4038, 'n_tokens_neox': 3867, 'n_words': 1559}