<commit_before>#! /usr/bin/python

import collectd
import json
import urllib2

VERBOSE_LOGGING = False

INSTANCE_TYPE_NAMENODE = 'namenode'
INSTANCE_TYPE_DATANODE = 'datanode'
INSTANCE_TYPE_HBASE_MASTER = 'hbase_master'

CONFIGS = []


BEAN_PREFIXES = {
    INSTANCE_TYPE_NAMENODE: {
        'FSNamesystem': 'Hadoop:service=NameNode,name=FSNamesystemState',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'Threading': 'java.lang:type=Threading',
        'JvmMetrics': 'Hadoop:service=NameNode,name=JvmMetrics',
    },
    INSTANCE_TYPE_DATANODE: {
        'DatanodeActivity': 'Hadoop:service=DataNode,name=DataNodeActivity',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'Threading': 'java.lang:type=Threading',
        'JvmMetrics': 'Hadoop:service=NameNode,name=JvmMetrics',
    },
    INSTANCE_TYPE_HBASE_MASTER: {
        'MasterBalancer': 'Hadoop:service=HBase,name=Master,sub=Balancer',
        'MasterAssignmentManager': 'Hadoop:service=HBase,name=Master,sub=AssignmentManger',
        'Threading': 'java.lang:type=Threading',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'MasterServer': 'Hadoop:service=HBase,name=Master,sub=Server',
        'JvmMetrics': 'Hadoop:service=HBase,name=JvmMetrics',
    },
}

def configure_callback(conf):
    """Received configuration information"""
    host = None
    port = None
    instance = None
    instance_type = None
    verbose_logging = VERBOSE_LOGGING

    for node in conf.children:
        if node.key == 'HDFSNamenodeHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_NAMENODE
        elif node.key == 'HDFSDatanodeHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_DATANODE
        elif node.key == 'HbaseMasterHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_HBASE_MASTER
        elif node.key == 'Port':
            port = node.values[0]
        elif node.key == 'Instance':
            instance = node.values[0]
        elif node.key == 'Verbose':
            verbose_logging = bool(node.values[0])
        else:
            collectd.warning('hadoop plugin: Unknown config key: %s.' % node.key)

    if not host and not instance and not instance_type and not port:
        collectd.error('hadoop plugin: Error Host, Port, Instance must be set.')
    else:
        config = {
            'host': host,
            'port': port,
            'instance': instance,
            'instance_type': instance_type,
            'verbose_logging': verbose_logging
        }

        CONFIGS.append(config)

        log_verbose('Configured hadoop instance_type=%s with host=%s' % (instance_type, host), verbose_logging)


def get_jmx_beans(host, port):
    jmx_url = "http://{}:{}/jmx".format(host, port)

    try:
        beans = json.load(urllib2.urlopen(jmx_url, timeout=10))['beans']
        return beans

    except urllib2.URLError as e:
        collectd.error('hadoop plugin: Error connecting to %s - %r' % (jmx_url, e))

    return {}


def process_metrics(host, port, instance, instance_type, verbose_logging):
    beans = get_jmx_beans(host, port)
<commit_msg>proper error conditions for invalid config<commit_after>#! /usr/bin/python

import collectd
import json
import urllib2

VERBOSE_LOGGING = False

INSTANCE_TYPE_NAMENODE = 'namenode'
INSTANCE_TYPE_DATANODE = 'datanode'
INSTANCE_TYPE_HBASE_MASTER = 'hbase_master'

CONFIGS = []


BEAN_PREFIXES = {
    INSTANCE_TYPE_NAMENODE: {
        'FSNamesystem': 'Hadoop:service=NameNode,name=FSNamesystemState',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'Threading': 'java.lang:type=Threading',
        'JvmMetrics': 'Hadoop:service=NameNode,name=JvmMetrics',
    },
    INSTANCE_TYPE_DATANODE: {
        'DatanodeActivity': 'Hadoop:service=DataNode,name=DataNodeActivity',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'Threading': 'java.lang:type=Threading',
        'JvmMetrics': 'Hadoop:service=NameNode,name=JvmMetrics',
    },
    INSTANCE_TYPE_HBASE_MASTER: {
        'MasterBalancer': 'Hadoop:service=HBase,name=Master,sub=Balancer',
        'MasterAssignmentManager': 'Hadoop:service=HBase,name=Master,sub=AssignmentManger',
        'Threading': 'java.lang:type=Threading',
        'OperatingSystem': 'java.lang:type=OperatingSystem',
        'MasterServer': 'Hadoop:service=HBase,name=Master,sub=Server',
        'JvmMetrics': 'Hadoop:service=HBase,name=JvmMetrics',
    },
}

def configure_callback(conf):
    """Received configuration information"""
    host = None
    port = None
    instance = None
    instance_type = None
    verbose_logging = VERBOSE_LOGGING

    for node in conf.children:
        if node.key == 'HDFSNamenodeHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_NAMENODE
        elif node.key == 'HDFSDatanodeHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_DATANODE
        elif node.key == 'HbaseMasterHost':
            host = node.values[0]
            instance_type = INSTANCE_TYPE_HBASE_MASTER
        elif node.key == 'Port':
            port = node.values[0]
        elif node.key == 'Instance':
            instance = node.values[0]
        elif node.key == 'Verbose':
            verbose_logging = bool(node.values[0])
        else:
            collectd.warning('hadoop plugin: Unknown config key: %s.' % node.key)

    if not host or not instance or not instance_type or not port:
        collectd.error('hadoop plugin error: *Host, Port, and Instance must be set.')
    else:
        config = {
            'host': host,
            'port': port,
            'instance': instance,
            'instance_type': instance_type,
            'verbose_logging': verbose_logging
        }

        CONFIGS.append(config)

        log_verbose('Configured hadoop instance_type=%s with host=%s' % (instance_type, host), verbose_logging)


def get_jmx_beans(host, port):
    jmx_url = "http://{}:{}/jmx".format(host, port)

    try:
        beans = json.load(urllib2.urlopen(jmx_url, timeout=10))['beans']
        return beans

    except urllib2.URLError as e:
        collectd.error('hadoop plugin: Error connecting to %s - %r' % (jmx_url, e))

    return {}


def process_metrics(host, port, instance, instance_type, verbose_logging):
    beans = get_jmx_beans(host, port)

{'fraction_non_alphanumeric': 0.11896201344380178, 'fraction_numerical': 0.003439112083789276, 'mean_word_length': 3.0725652450668366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2420206', 'n_tokens_mistral': 2065, 'n_tokens_neox': 2025, 'n_words': 384}