<commit_before># Copyright 2014 IBM Corporation
# Copyright 2015-2017 Lenovo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
import confluent.exceptions as exc
import confluent.firmwaremanager as firmwaremanager
import confluent.interface.console as conapi
import confluent.messages as msg
import confluent.util as util
import eventlet
import eventlet.event
import eventlet.green.threading as threading
import eventlet.greenpool as greenpool
import eventlet.queue as queue
import eventlet.support.greendns
from fnmatch import fnmatch
import pyghmi.constants as pygconstants
import pyghmi.exceptions as pygexc
console = eventlet.import_patched('pyghmi.ipmi.console')
ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
import socket
import ssl

pci_cache = {}

def get_dns_txt(qstring):
    return eventlet.support.greendns.resolver.query(
        qstring, 'TXT')[0].strings[0].replace('i=', '')

def get_pci_text_from_ids(subdevice, subvendor, device, vendor):
    fqpi = '{0}.{1}.{2}.{3}'.format(subdevice, subvendor, device, vendor)
    if fqpi in pci_cache:
        return pci_cache[fqpi]
    vendorstr = None
    try:
        vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(subvendor))
    except Exception:
        try:
            vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(vendor))
        except Exception:
            pass
    devstr = None
    try:
        devstr = get_dns_txt(fqpi + '.pci.id.ucw.cz')
    except Exception:
        try:
            devstr = get_dns_txt('{0}.{1}.pci.id.ucw.cz'.format(
                device, vendor))
        except Exception:
            pass
    if vendorstr and devstr:
        pci_cache[fqpi] = vendorstr, devstr
    return vendorstr, devstr


# There is something not right with the RLocks used in pyghmi when
# eventlet comes into play.  It seems like sometimes on acquire,
# it calls _get_ident and it isn't the id(greenlet) and so
# a thread deadlocks itself due to identity crisis?
# However, since we are not really threaded, the operations being protected
# are not actually dangerously multiplexed...  so we can replace with
# a null context manager for now
class NullLock(object):

    def donothing(self, *args, **kwargs):
        return 1

    __enter__ = donothing
    __exit__ = donothing
    acquire = donothing
    release = donothing

console.session.select = eventlet.green.select
console.session.threading = eventlet.green.threading
console.session.WAITING_SESSIONS = NullLock()
console.session.KEEPALIVE_SESSIONS = NullLock()
console.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo


def exithandler():
    if console.session.iothread is not None:
        console.session.iothread.join()

atexit.register(exithandler)

_ipmiworkers = greenpool.GreenPool()

_ipmithread = None
_ipmiwaiters = []

sensor_categories = {
    'temperature': frozenset(['Temperature']),
    'energy': frozenset(['Energy']),
    'power': frozenset(['Power', 'Current']),
    'fans': frozenset(['Fan', 'Cooling Device']),
}


class EmptySensor(object):
    def __init__(self, name):
        self.name = name
        self.value = None
        self.states = ['Unavailable']
        self.units = None
        self.health = 'ok'


def hex2bin(hexstring):
    hexvals = hexstring.split(':')
    if len(hexvals) < 2:
        hexvals = hexstring.split(' ')
    if len(hexvals) < 2:
        hexvals = [hexstring[i:i+2] for i in xrange(0, len(hexstring), 2)]
    bytedata = [int(i, 16) for i in hexvals]
    return bytearray(bytedata)


def simplify_name(name):
    return name.lower().replace(' ', '_').replace('/', '-')


def sanitize_invdata(indata):
    """Sanitize pyghmi data

    pyghmi will return bytearrays when it has no idea what to do.  In our
    case, we will change those to hex strings.  Additionally, ignore 'extra'
    fields if the oem_parser is set
    """
    if 'oem_parser' in indata and indata['oem_parser'] is not None:
        if 'board_extra' in indata:
            del indata['board_extra']
        if 'chassis_extra' in indata:
            del indata['chassis_extra']
        if 'product_extra' in indata:
            del indata['product_extra']
    for k in indata:
        if isinstance(indata[k], bytearray):
            indata[k] = '0x' + ''.join(format(x, '02x') for x in indata[k])
        elif isinstance(indata[k], dict):
            sanitize_invdata(indata[k])
        elif isinstance(indata[k], list):
            for idx, value in enumerate(indata[k]):
                if isinstance(value, bytearray):
                    indata[k][idx] = '0x' + ''.join(
                        format(x, '02x') for x in indata[k][idx])


class IpmiCommandWrapper(ipmicommand.Command):
    def __init__(self, node, cfm, **kwargs):
        self.cfm = cfm
        self.node = node
        self._inhealth = False
        self._lasthealth = None
        self._attribwatcher = cfm.watch_attributes(
            (node,), ('secret.hardwaremanagementuser', 'collective.manager',
                      'secret.hardwaremanagementpassword', 'secret.ipmikg',
                      'hardwaremanagement.manager'), self._attribschanged)
        super(self.__class__, self).__init__(**kwargs)
        self.setup_confluent_keyhandler()

    def setup_confluent_keyhandler(self):
        self.register_key_handler(util.TLSCertVerifier(
            self.cfm, self.node, 'pubkeys.tls_hardwaremanager').verify_cert)

    def close_confluent(self):
        if self._attribwatcher:
            self.cfm.remove_watcher(self._attribwatcher)
            self._attribwatcher = None

    def _attribschanged(self, nodeattribs, configmanager, **kwargs):
        try:
            self.ipmi_session._mark_broken()
        except AttributeError:
            # if ipmi_session doesn't already exist,
            # then do nothing
            pass

    def get_health(self):
        if self._inhealth:
            while self._inhealth:
                eventlet.sleep(0.1)
            return self._lasthealth
        self._inhealth = True
        try:
            self._lasthealth = super(IpmiCommandWrapper, self).get_health()
        except Exception:
            self._inhealth = False
            raise
        self._inhealth = False
        return self._lasthealth


def _ipmi_evtloop():
    while True:
        try:
            console.session.Session.wait_for_rsp(timeout=600)
            while _ipmiwaiters:
                waiter = _ipmiwaiters.pop()
                waiter.send()
        except:  # TODO(jbjohnso): log the trace into the log
            import traceback

            traceback.print_exc()


def get_conn_params(node, configdata):
    if 'secret.hardwaremanagementuser' in configdata:
        username = configdata['secret.hardwaremanagementuser']['value']
    else:
        username = 'USERID'
    if 'secret.hardwaremanagementpassword' in configdata:
        passphrase = configdata['secret.hardwaremanagementpassword']['value']
    else:
        passphrase = '<PASSWORD>'  # for lack of a better guess
    if 'hardwaremanagement.manager' in configdata:
        bmc = configdata['hardwaremanagement.manager']['value']
    else:
        bmc = node
    if 'secret.ipmikg' in configdata:
        kg = configdata['secret.ipmikg']['value']
    else:
        kg = passphrase
    # TODO(jbjohnso): check if the end has some number after a : without []
    # for non default port
    return {
        'username': username,
        'passphrase': <PASSWORD>,
        'kg': kg,
        'bmc': bmc,
        'port': 623,
    }


_configattributes = ('secret.hardwaremanagementuser',
                     'secret.hardwaremanagementpassword',
                     'secret.ipmikg', 'hardwaremanagement.manager')


def _donothing(data):
    # a dummy function to avoid some awkward exceptions from
    # zombie pyghmi console objects
    pass


class IpmiConsole(conapi.Console):
    configattributes = frozenset(_configattributes)

    def __init__(self, node, config):
        self.error = None
        self.datacallback = None
        crypt = config.decrypt
        self.solconnection = None
        config.decrypt = True
        self.broken = False
        configdata = config.get_node_attributes([node], _configattributes)
        connparams = get_conn_params(node, configdata[node])
        config.decrypt = crypt
        self.username = connparams['username']
        self.password = connparams['<PASSWORD>phrase']
        self.kg = connparams['kg']
        self.bmc = connparams['bmc']
        self.port = connparams['port']
        self.connected = False
        # Cannot actually create console until 'connect', when we get callback

    def __del__(self):
        self.solconnection = None

    def handle_data(self, data):
        if type(data) == dict:
            if 'error' in data:
                self.solconnection = None
                self.broken = True
                self.error = data['error']
                if self.connected:
                    self.connected = False
                    self.datacallback(conapi.ConsoleEvent.Disconnect)
        else:
            self.datacallback(data)

    def connect(self, callback):
        self.datacallback = callback
        # we provide a weak reference to pyghmi as otherwise we'd
        # have a circular reference and reference counting would never get
        # out...
        try:
            self.solconnection = console.Console(bmc=self.bmc, port=self.port,
                                                 userid=self.username,
                                                 password=self.password,
                                                 kg=self.kg, force=True,
                                                 iohandler=self.handle_data)
            self.solconnection.outputlock = NullLock()
            while (self.solconnection and not self.solconnection.connected and
                   not (self.broken or self.solconnection.broken or
                        self.solconnection.ipmi_session.broken)):
                w = eventlet.event.Event()
                _ipmiwaiters.append(w)
                w.wait(15)
            if (self.broken or not self.solconnection or
                    self.solconnection.broken or
                    self.solconnection.ipmi_session.broken):
                if not self.error:
                    self.error = 'Unknown error'
                if (self.error.startswith('Incorrect password') or
                        self.error.startswith('Unauthorized name')):
                    raise exc.TargetEndpointBadCredentials
                else:
                    raise exc.TargetEndpointUnreachable(self.error)
            self.connected = True
        except socket.gaierror as err:
            raise exc.TargetEndpointUnreachable(str(err))

    def close(self):
        if self.solconnection is not None:
            # break the circular reference here
            self.solconnection.out_handler = _donothing
            self.solconnection.close()
        self.solconnection = None
        self.datacallback = None
        self.broken = True
        self.error = "closed"

    def write(self, data):
        self.solconnection.send_data(data)

    def send_break(self):
        self.solconnection.send_break()


def perform_requests(operator, nodes, element, cfg, inputdata):
    cryptit = cfg.decrypt
    cfg.decrypt = True
    configdata = cfg.get_node_attributes(nodes, _configattributes)
    cfg.decrypt = cryptit
    resultdata = queue.LightQueue()
    livingthreads = set([])
    for node in nodes:
        livingthreads.add(_ipmiworkers.spawn(
            perform_request, operator, node, element, configdata, inputdata,
            cfg, resultdata))
    while livingthreads:
        try:
            datum = resultdata.get(timeout=10)
            while datum:
                if datum != 'Done':
                    if isinstance(datum, Exception):
                        raise datum
                    yield datum
                datum = resultdata.get_nowait()
        except queue.Empty:
            pass
        for t in list(livingthreads):
            if t.dead:
                livingthreads.discard(t)
    try:
        # drain queue if a thread put something on the queue and died
        while True:
            datum = resultdata.get_nowait()
            if datum != 'Done':
                yield datum
    except queue.Empty:
        pass


def perform_request(operator, node, element,
                    configdata, inputdata, cfg, results):
        try:
            return IpmiHandler(operator, node, element, configdata, inputdata,
                               cfg, results).handle_request()
        except pygexc.IpmiException as ipmiexc:
            excmsg = str(ipmiexc)
            if excmsg in ('Session no longer connected', 'timeout'):
                results.put(msg.ConfluentTargetTimeout(node))
            else:
                results.put(msg.ConfluentNodeError(node, excmsg))
                raise
        except exc.TargetEndpointUnreachable as tu:
            results.put(msg.ConfluentTargetTimeout(node, str(tu)))
        except ssl.SSLEOFError:
            results.put(msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC'))
        except exc.PubkeyInvalid:
            results.put(msg.ConfluentNodeError(
                node,
                'Mismatch detected between target certificate fingerprint '
                'and pubkeys.tls_hardwaremanager attribute'))
        except pygexc.InvalidParameterValue as e:
            results.put(msg.ConfluentNodeError(node, str(e)))
        except Exception as e:
            results.put(e)
            raise
        finally:
            results.put('Done')

persistent_ipmicmds = {}

class IpmiHandler(object):
    def __init__(self, operation, node, element, cfd, inputdata, cfg, output):
        self.sensormap = {}
        self.invmap = {}
        self.output = output
        self.sensorcategory = None
        self.broken = False
        self.error = None
        eventlet.sleep(0)
        self.cfg = cfd[node]
        self.current_user = cfg.current_user
        self.loggedin = False
        self.node = node
        self.element = element
        self.op = operation
        connparams = get_conn_params(node, self.cfg)
        self.ipmicmd = None
        self.inputdata = inputdata
        self.tenant = cfg.tenant
        tenant = cfg.tenant
        if ((node, tenant) not in persistent_ipmicmds or
                not persistent_ipmicmds[(node, tenant)].ipmi_session.logged or
                persistent_ipmicmds[(node, tenant)].ipmi_session.broken):
            try:
                persistent_ipmicmds[(node, tenant)].close_confluent()
            except KeyError:  # was no previous session
                pass
            try:
                persistent_ipmicmds[(node, tenant)] = IpmiCommandWrapper(
                    node, cfg, bmc=connparams['bmc'],
                    userid=connparams['username'],
                    password=<PASSWORD>params['<PASSWORD>'], kg=connparams['kg'],
                    port=connparams['port'], onlogon=self.logged)

                ipmisess = persistent_ipmicmds[(node, tenant)].ipmi_session
                begin = util.monotonic_time()
                while ((not (self.broken or self.loggedin)) and
                               (util.monotonic_time() - begin) < 30):
                    ipmisess.wait_for_rsp(31 - (util.monotonic_time() - begin))
                if not (self.broken or self.loggedin):
                    ipmisess._mark_broken()
                    raise exc.TargetEndpointUnreachable(
                        "Login process to " + connparams['bmc'] + " died")
            except socket.gaierror as ge:
                if ge[0] == -2:
                    raise exc.TargetEndpointUnreachable(ge[1])
                raise
        self.ipmicmd = persistent_ipmicmds[(node, tenant)]

    bootdevices = {
        'optical': 'cd'
    }

    def logged(self, response, ipmicmd):
        if 'error' in response:
            self.broken = True
            self.error = response['error']
        else:
            self.ipmicmd = ipmicmd
            self.loggedin = True

    def handle_request(self):
        if self.broken:
            if (self.error == 'timeout' or
                    'Insufficient resources' in self.error):
                self.error = self.error.replace(' reported in RAKP4', '')
                self.output.put(msg.ConfluentTargetTimeout(
                    self.node, self.error))
                return
            elif ('Unauthorized' in self.error or
                    'Incorrect password' in self.error):
                self.output.put(
                    msg.ConfluentTargetInvalidCredentials(self.node))
                return
            else:
                raise Exception(self.error)
        if self.element == ['power', 'state']:
            self.power()
        elif self.element == ['_enclosure', 'reseat_bay']:
            self.reseat_bay()
        elif self.element == ['boot', 'nextdevice']:
            self.bootdevice()
        elif self.element == ['health', 'hardware']:
            self.health()
        elif self.element == ['identify']:
            self.identify()
        elif self.element[0] == 'sensors':
            self.handle_sensors()
        elif self.element[0] == 'configuration':
            self.handle_configuration()
        elif self.element[:3] == ['inventory', 'firmware', 'updates']:
            self.handle_update()
        elif self.element[0] == 'inventory':
            self.handle_inventory()
        elif self.element == ['media', 'attach']:
            self.handle_attach_media()
        elif self.element == ['media', 'detach']:
            self.handle_detach_media()
        elif self.element == ['media', 'uploads']:
            self.handle_media_upload()
        elif self.element == ['media', 'current']:
            self.handle_list_media()
        elif self.element == ['events', 'hardware', 'log']:
            self.do_eventlog()
        elif self.element == ['events', 'hardware', 'decode']:
            self.decode_alert()
        elif self.element == ['console', 'license']:
            self.handle_license()
        elif self.element == ['support', 'servicedata']:
            self.handle_servicedata_fetch()
        elif self.element == ['description']:
            self.handle_description()
        else:
            raise Exception('Not Implemented')

    def handle_update(self):
        u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                    self.inputdata.filename, self.tenant,
                                    bank=self.inputdata.bank)
        self.output.put(
            msg.CreatedResource(
                'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
                    self.node, u.name)))

    def handle_media_upload(self):
        u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                     self.inputdata.filename, self.tenant,
                                     type='mediaupload')
        self.output.put(msg.CreatedResource(
            'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))

    def handle_servicedata_fetch(self):
        u = firmwaremanager.Updater(
            self.node, self.ipmicmd.get_diagnostic_data,
            self.inputdata.filename, self.tenant, type='ffdc',
            owner=self.current_user)
        self.output.put(msg.CreatedResource(
            'nodes/{0}/support/servicedata/{1}'.format(self.node, u.name)))

    def handle_attach_media(self):
        try:
            self.ipmicmd.attach_remote_media(self.inputdata.filename)
        except pygexc.UnsupportedFunctionality as uf:
            self.output.put(msg.ConfluentNodeError(self.node, str(uf)))

    def handle_detach_media(self):
        self.ipmicmd.detach_remote_media()

    def handle_list_media(self):
        for media in self.ipmicmd.list_media():
            self.output.put(msg.Media(self.node, media))

    def handle_configuration(self):
        if self.element[1:3] == ['management_controller', 'alerts']:
            return self.handle_alerts()
        elif self.element[1:3] == ['management_controller', 'users']:
            return self.handle_users()
        elif self.element[1:3] == ['management_controller', 'net_interfaces']:
            return self.handle_nets()
        elif self.element[1:3] == ['management_controller', 'reset']:
            return self.handle_reset()
        elif self.element[1:3] == ['management_controller', 'identifier']:
            return self.handle_identifier()
        elif self.element[1:3] == ['management_controller', 'hostname']:
            return self.handle_hostname()
        elif self.element[1:3] == ['management_controller', 'domain_name']:
            return self.handle_domain_name()
        elif self.element[1:3] == ['management_controller', 'ntp']:
            return self.handle_ntp()
        elif self.element[1:3] == ['system', 'all']:
            return self.handle_sysconfig()
        elif self.element[1:3] == ['system', 'advanced']:
            return self.handle_sysconfig(True)
        elif self.element[1:3] == ['system', 'clear']:
            return self.handle_sysconfigclear()
        raise Exception('Not implemented')

    def decode_alert(self):
        inputdata = self.inputdata.get_alert(self.node)
        specifictrap = int(inputdata['.1.3.6.1.6.3.1.1.4.1.0'].rpartition(
            '.')[-1])
        for tmpvarbind in inputdata:
            if tmpvarbind.endswith('3183.1.1'):
                varbinddata = inputdata[tmpvarbind]
        varbinddata = hex2bin(varbinddata)
        event = self.ipmicmd.decode_pet(specifictrap, varbinddata)
        self.pyghmi_event_to_confluent(event)
        self.output.put(msg.EventCollection((event,), name=self.node))

    def handle_alerts(self):
        if self.element[3] == 'destinations':
            if len(self.element) == 4:
                # A list of destinations
                maxdest = self.ipmicmd.get_alert_destination_count()
                for alertidx in xrange(0, maxdest + 1):
                    self.output.put(msg.ChildCollection(alertidx))
                return
            elif len(self.element) == 5:
                alertidx = int(self.element[-1])
                if self.op == 'read':
                    destdata = self.ipmicmd.get_alert_destination(alertidx)
                    self.output.put(msg.AlertDestination(
                        ip=destdata['address'],
                        acknowledge=destdata['acknowledge_required'],
                        acknowledge_timeout=destdata.get('acknowledge_timeout', None),
                        retries=destdata['retries'],
                        name=self.node))
                    return
                elif self.op == 'update':
                    alertparms = self.inputdata.alert_params_by_node(
                        self.node)
                    alertargs = {}
                    if 'acknowledge' in alertparms:
                        alertargs['acknowledge_required'] = alertparms['acknowledge']
                    if 'acknowledge_timeout' in alertparms:
                        alertargs['acknowledge_timeout'] = alertparms['acknowledge_timeout']
                    if 'ip' in alertparms:
                        alertargs['ip'] = alertparms['ip']
                    if 'retries' in alertparms:
                        alertargs['retries'] = alertparms['retries']
                    self.ipmicmd.set_alert_destination(destination=alertidx,
                                                       **alertargs)
                    return
                elif self.op == 'delete':
                    self.ipmicmd.clear_alert_destination(alertidx)
                    return
        raise Exception('Not implemented')

    def handle_nets(self):
        if len(self.element) == 3:
            if self.op != 'read':
                self.output.put(
                    msg.ConfluentNodeError(self.node, 'Unsupported operation'))
                return
            self.output.put(msg.ChildCollection('management'))
        elif len(self.element) == 4 and self.element[-1] == 'management':
            if self.op == 'read':
                lancfg = self.ipmicmd.get_net_configuration()
                self.output.put(msg.NetworkConfiguration(
                    self.node, ipv4addr=lancfg['ipv4_address'],
                    ipv4gateway=lancfg['ipv4_gateway'],
                    ipv4cfgmethod=lancfg['ipv4_configuration'],
                    hwaddr=lancfg['mac_address']
                ))
            elif self.op == 'update':
                config = self.inputdata.netconfig(self.node)
                try:
                    self.ipmicmd.set_net_configuration(
                        ipv4_address=config['ipv4_address'],
                        ipv4_configuration=config['ipv4_configuration'],
                        ipv4_gateway=config['ipv4_gateway'])
                except socket.error as se:
                    self.output.put(msg.ConfluentNodeError(self.node,
                                                           se.message))
                except ValueError as e:
                    if e.message == 'negative shift count':
                        self.output.put(msg.ConfluentNodeError(
                            self.node, 'Invalid prefix length given'))
                    else:
                        raise

    def handle_users(self):
        # Create user
        if len(self.element) == 3:
            if self.op == 'update':
                user = self.inputdata.credentials[self.node]
                self.ipmicmd.create_user(uid=user['uid'], name=user['username'],
                                    password=<PASSWORD>['password'],
                                    callback=True,link_auth=True, ipmi_msg=True,
                                    privilege_level=user['privilege_level'])
            # A list of users
            self.output.put(msg.ChildCollection('all'))
            for user in self.ipmicmd.get_users():
                self.output.put(msg.ChildCollection(user, candelete=True))
            return
        # List all users
        elif len(self.element) == 4 and self.element[-1] == 'all':
            users = []
            for user in self.ipmicmd.get_users():
                users.append(self.ipmicmd.get_user(uid=user))
            self.output.put(msg.UserCollection(users=users, name=self.node))
            return
        # Update user
        elif len(self.element) == 4:
            user = int(self.element[-1])
            if self.op == 'read':
                data = self.ipmicmd.get_user(uid=user)
                self.output.put(msg.User(
                    uid=data['uid'],
                    username=data['name'],
                    privilege_level=data['access']['privilege_level'],
                    name=self.node))
                return
            elif self.op == 'update':
                user = self.inputdata.credentials[self.node]

                if 'username' in user:
                    self.ipmicmd.set_user_name(uid=user['uid'],
                                               name=user['username'])
                if 'privilege_level' in user:
                    self.ipmicmd.set_user_access(uid=user['uid'],
                                    privilege_level=user['privilege_level'])
                if 'password' in user:
                    self.ipmicmd.set_user_password(uid=user['uid'],
                                                   password=user['password'])
                    self.ipmicmd.set_user_password(uid=user['uid'],
                                    mode='enable', password=user['password'])
                if 'enabled' in user:
                    if user['enabled'] == 'yes':
                        mode = 'enable'
                    else:
                        mode = 'disable'
                    self.ipmicmd.disable_user(user['uid'], mode)
                return
            elif self.op == 'delete':
                self.ipmicmd.user_delete(uid=user)
                return

    def do_eventlog(self):
        eventout = []
        clear = False
        if self.op == 'delete':
            clear = True
        for event in self.ipmicmd.get_event_log(clear):
            self.pyghmi_event_to_confluent(event)
            eventout.append(event)
        self.output.put(msg.EventCollection(eventout, name=self.node))

    def pyghmi_event_to_confluent(self, event):
        event['severity'] = _str_health(event.get('severity', 'unknown'))
        if 'event_data' in event:
            event['event'] = '{0} - {1}'.format(
                event['event'], event['event_data'])
        if 'event_id' in event:
            event['id'] = '{0}.{1}'.format(event['event_id'],
                                           event['component_type_id'])

    def make_inventory_map(self):
        invnames = self.ipmicmd.get_inventory_descriptions()
        for name in invnames:
            self.invmap[simplify_name(name)] = name

    def make_sensor_map(self, sensors=None):
        if sensors is None:
            sensors = self.ipmicmd.get_sensor_descriptions()
        for sensor in sensors:
            resourcename = sensor['name']
            self.sensormap[simplify_name(resourcename)] = resourcename

    def read_sensors(self, sensorname):
        if sensorname == 'all':
            sensors = self.ipmicmd.get_sensor_descriptions()
            readings = []
            for sensor in filter(self.match_sensor, sensors):
                try:
                    reading = self.ipmicmd.get_sensor_reading(
                        sensor['name'])
                except pygexc.IpmiException as ie:
                    if ie.ipmicode == 203:
                        self.output.put(msg.SensorReadings([EmptySensor(
                            sensor['name'])], name=self.node))
                        continue
                    raise
                if hasattr(reading, 'health'):
                    reading.health = _str_health(reading.health)
                readings.append(reading)
            self.output.put(msg.SensorReadings(readings, name=self.node))
        else:
            self.make_sensor_map()
            if sensorname not in self.sensormap:
                self.output.put(
                    msg.ConfluentTargetNotFound(self.node,
                                                'Sensor not found'))
                return
            try:
                reading = self.ipmicmd.get_sensor_reading(
                    self.sensormap[sensorname])
                if hasattr(reading, 'health'):
                    reading.health = _str_health(reading.health)
                self.output.put(
                    msg.SensorReadings([reading],
                                       name=self.node))
            except pygexc.IpmiException as ie:
                if ie.ipmicode == 203:
                    self.output.put(msg.ConfluentResourceUnavailable(
                        self.node, 'Unavailable'
                    ))
                else:
                    self.output.put(msg.ConfluentTargetTimeout(self.node))

    def list_inventory(self):
        try:
            components = self.ipmicmd.get_inventory_descriptions()
        except pygexc.IpmiException:
            self.output.put(msg.ConfluentTargetTimeout(self.node))
            return
        self.output.put(msg.ChildCollection('all'))
        for component in components:
            self.output.put(msg.ChildCollection(simplify_name(component)))

    def list_firmware(self):
        self.output.put(msg.ChildCollection('all'))
        for id, data in self.ipmicmd.get_firmware():
            self.output.put(msg.ChildCollection(simplify_name(id)))

    def read_firmware(self, component):
        items = []
        errorneeded = False
        try:
            complist = () if component == 'all' else (component,)
            for id, data in self.ipmicmd.get_firmware(complist):
                if (component in ('core', 'all') or
                        component == simplify_name(id)):
                    items.append({id: data})
        except ssl.SSLEOFError:
            errorneeded = msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC while trying to read extended '
                           'information')
        except exc.PubkeyInvalid:
            errorneeded = msg.ConfluentNodeError(
                self.node,
                'Extended information unavailable, mismatch detected between '
                'target certificate fingerprint and '
                'pubkeys.tls_hardwaremanager attribute')
        self.output.put(msg.Firmware(items, self.node))
        if errorneeded:
            self.output.put(errorneeded)

    def handle_inventory(self):
        if self.element[1] == 'firmware':
            if len(self.element) == 3:
                return self.list_firmware()
            elif len(self.element) == 4:
                return self.read_firmware(self.element[-1])
        elif self.element[1] == 'hardware':
            if len(self.element) == 3:  # list things in inventory
                return self.list_inventory()
            elif len(self.element) == 4:  # actually read inventory data
                return self.read_inventory(self.element[-1])
        raise Exception('Unsupported scenario...')

    def list_leds(self):
        self.output.put(msg.ChildCollection('all'))
        for category, info in self.ipmicmd.get_leds():
            self.output.put(msg.ChildCollection(simplify_name(category)))

    def read_leds(self, component):
        led_categories = []
        for category, info in self.ipmicmd.get_leds():
            if component == 'all' or component == simplify_name(category):
                led_categories.append({category: info})
        self.output.put(msg.LEDStatus(led_categories, self.node))

    def read_inventory(self, component):
        errorneeded = False
        try:
            invitems = []
            if component == 'all':
                for invdata in self.ipmicmd.get_inventory():
                    if invdata[1] is None:
                        newinf = {'present': False, 'information': None}
                    else:
                        sanitize_invdata(invdata[1])
                        newinf = {'present': True, 'information': invdata[1]}
                    newinf['name'] = invdata[0]
                    self.add_invitem(invitems, newinf)
            else:
                self.make_inventory_map()
                compname = self.invmap.get(component, None)
                if compname is None:
                    self.output.put(msg.ConfluentTargetNotFound())
                    return
                invdata = self.ipmicmd.get_inventory_of_component(compname)
                if invdata is None:
                    newinf = {'present': False, 'information': None}
                else:
                    sanitize_invdata(invdata)
                    newinf = {'present': True, 'information': invdata}
                newinf['name'] = compname
                self.add_invitem(invitems, newinf)
        except ssl.SSLEOFError:
            errorneeded = msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC while trying to read extended '
                           'information')
        except exc.PubkeyInvalid:
            errorneeded = msg.ConfluentNodeError(
                self.node,
                'Extended information unavailable, mismatch detected between '
                'target certificate fingerprint and '
                'pubkeys.tls_hardwaremanager attribute')
        newinvdata = {'inventory': invitems}
        self.output.put(msg.KeyValueData(newinvdata, self.node))
        if errorneeded:
            self.output.put(errorneeded)

    def add_invitem(self, invitems, newinf):
        if fnmatch(newinf['name'], 'Adapter ??:??:??') or fnmatch(
                newinf['name'], 'PCIeGen? x*'):
            myinf = newinf.get('information', {})
            sdid = myinf.get('PCI Subsystem Device ID', None)
            svid = myinf.get('PCI Subsystem Vendor ID', None)
            did = myinf.get('PCI Device ID', None)
            vid = myinf.get('PCI Vendor ID', None)
            vstr, dstr = get_pci_text_from_ids(sdid, svid, did, vid)
            if vstr:
                newinf['information']['PCI Vendor'] = vstr
            if dstr:
                newinf['name'] = dstr
        invitems.append(newinf)

    def handle_sensors(self):
        if self.element[-1] == '':
            self.element = self.element[:-1]
        if len(self.element) < 3:
            return
        self.sensorcategory = self.element[2]
        # list sensors per category
        if len(self.element) == 3 and self.element[-2] == 'hardware':
            if self.sensorcategory == 'leds':
                return self.list_leds()
            return self.list_sensors()
        elif len(self.element) == 4:  # resource requested
            if self.sensorcategory == 'leds':
                return self.read_leds(self.element[-1])
            return self.read_sensors(self.element[-1])

    def match_sensor(self, sensor):
        if self.sensorcategory == 'all':
            return True<commit_msg>Fix non-unique name for similar inventory items.<commit_after># Copyright 2014 IBM Corporation
# Copyright 2015-2017 Lenovo
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import atexit
import confluent.exceptions as exc
import confluent.firmwaremanager as firmwaremanager
import confluent.interface.console as conapi
import confluent.messages as msg
import confluent.util as util
import eventlet
import eventlet.event
import eventlet.green.threading as threading
import eventlet.greenpool as greenpool
import eventlet.queue as queue
import eventlet.support.greendns
from fnmatch import fnmatch
import pyghmi.constants as pygconstants
import pyghmi.exceptions as pygexc
console = eventlet.import_patched('pyghmi.ipmi.console')
ipmicommand = eventlet.import_patched('pyghmi.ipmi.command')
import socket
import ssl

pci_cache = {}

def get_dns_txt(qstring):
    return eventlet.support.greendns.resolver.query(
        qstring, 'TXT')[0].strings[0].replace('i=', '')

def get_pci_text_from_ids(subdevice, subvendor, device, vendor):
    fqpi = '{0}.{1}.{2}.{3}'.format(subdevice, subvendor, device, vendor)
    if fqpi in pci_cache:
        return pci_cache[fqpi]
    vendorstr = None
    try:
        vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(subvendor))
    except Exception:
        try:
            vendorstr = get_dns_txt('{0}.pci.id.ucw.cz'.format(vendor))
        except Exception:
            pass
    devstr = None
    try:
        devstr = get_dns_txt(fqpi + '.pci.id.ucw.cz')
    except Exception:
        try:
            devstr = get_dns_txt('{0}.{1}.pci.id.ucw.cz'.format(
                device, vendor))
        except Exception:
            pass
    if vendorstr and devstr:
        pci_cache[fqpi] = vendorstr, devstr
    return vendorstr, devstr


# There is something not right with the RLocks used in pyghmi when
# eventlet comes into play.  It seems like sometimes on acquire,
# it calls _get_ident and it isn't the id(greenlet) and so
# a thread deadlocks itself due to identity crisis?
# However, since we are not really threaded, the operations being protected
# are not actually dangerously multiplexed...  so we can replace with
# a null context manager for now
class NullLock(object):

    def donothing(self, *args, **kwargs):
        return 1

    __enter__ = donothing
    __exit__ = donothing
    acquire = donothing
    release = donothing

console.session.select = eventlet.green.select
console.session.threading = eventlet.green.threading
console.session.WAITING_SESSIONS = NullLock()
console.session.KEEPALIVE_SESSIONS = NullLock()
console.session.socket.getaddrinfo = eventlet.support.greendns.getaddrinfo


def exithandler():
    if console.session.iothread is not None:
        console.session.iothread.join()

atexit.register(exithandler)

_ipmiworkers = greenpool.GreenPool()

_ipmithread = None
_ipmiwaiters = []

sensor_categories = {
    'temperature': frozenset(['Temperature']),
    'energy': frozenset(['Energy']),
    'power': frozenset(['Power', 'Current']),
    'fans': frozenset(['Fan', 'Cooling Device']),
}


class EmptySensor(object):
    def __init__(self, name):
        self.name = name
        self.value = None
        self.states = ['Unavailable']
        self.units = None
        self.health = 'ok'


def hex2bin(hexstring):
    hexvals = hexstring.split(':')
    if len(hexvals) < 2:
        hexvals = hexstring.split(' ')
    if len(hexvals) < 2:
        hexvals = [hexstring[i:i+2] for i in xrange(0, len(hexstring), 2)]
    bytedata = [int(i, 16) for i in hexvals]
    return bytearray(bytedata)


def simplify_name(name):
    return name.lower().replace(' ', '_').replace('/', '-')


def sanitize_invdata(indata):
    """Sanitize pyghmi data

    pyghmi will return bytearrays when it has no idea what to do.  In our
    case, we will change those to hex strings.  Additionally, ignore 'extra'
    fields if the oem_parser is set
    """
    if 'oem_parser' in indata and indata['oem_parser'] is not None:
        if 'board_extra' in indata:
            del indata['board_extra']
        if 'chassis_extra' in indata:
            del indata['chassis_extra']
        if 'product_extra' in indata:
            del indata['product_extra']
    for k in indata:
        if isinstance(indata[k], bytearray):
            indata[k] = '0x' + ''.join(format(x, '02x') for x in indata[k])
        elif isinstance(indata[k], dict):
            sanitize_invdata(indata[k])
        elif isinstance(indata[k], list):
            for idx, value in enumerate(indata[k]):
                if isinstance(value, bytearray):
                    indata[k][idx] = '0x' + ''.join(
                        format(x, '02x') for x in indata[k][idx])


class IpmiCommandWrapper(ipmicommand.Command):
    def __init__(self, node, cfm, **kwargs):
        self.cfm = cfm
        self.node = node
        self._inhealth = False
        self._lasthealth = None
        self._attribwatcher = cfm.watch_attributes(
            (node,), ('secret.hardwaremanagementuser', 'collective.manager',
                      'secret.hardwaremanagementpassword', 'secret.ipmikg',
                      'hardwaremanagement.manager'), self._attribschanged)
        super(self.__class__, self).__init__(**kwargs)
        self.setup_confluent_keyhandler()

    def setup_confluent_keyhandler(self):
        self.register_key_handler(util.TLSCertVerifier(
            self.cfm, self.node, 'pubkeys.tls_hardwaremanager').verify_cert)

    def close_confluent(self):
        if self._attribwatcher:
            self.cfm.remove_watcher(self._attribwatcher)
            self._attribwatcher = None

    def _attribschanged(self, nodeattribs, configmanager, **kwargs):
        try:
            self.ipmi_session._mark_broken()
        except AttributeError:
            # if ipmi_session doesn't already exist,
            # then do nothing
            pass

    def get_health(self):
        if self._inhealth:
            while self._inhealth:
                eventlet.sleep(0.1)
            return self._lasthealth
        self._inhealth = True
        try:
            self._lasthealth = super(IpmiCommandWrapper, self).get_health()
        except Exception:
            self._inhealth = False
            raise
        self._inhealth = False
        return self._lasthealth


def _ipmi_evtloop():
    while True:
        try:
            console.session.Session.wait_for_rsp(timeout=600)
            while _ipmiwaiters:
                waiter = _ipmiwaiters.pop()
                waiter.send()
        except:  # TODO(jbjohnso): log the trace into the log
            import traceback

            traceback.print_exc()


def get_conn_params(node, configdata):
    if 'secret.hardwaremanagementuser' in configdata:
        username = configdata['secret.hardwaremanagementuser']['value']
    else:
        username = 'USERID'
    if 'secret.hardwaremanagementpassword' in configdata:
        passphrase = configdata['secret.hardwaremanagementpassword']['value']
    else:
        passphrase = '<PASSWORD>'  # for lack of a better guess
    if 'hardwaremanagement.manager' in configdata:
        bmc = configdata['hardwaremanagement.manager']['value']
    else:
        bmc = node
    if 'secret.ipmikg' in configdata:
        kg = configdata['secret.ipmikg']['value']
    else:
        kg = passphrase
    # TODO(jbjohnso): check if the end has some number after a : without []
    # for non default port
    return {
        'username': username,
        'passphrase': passphrase,
        'kg': kg,
        'bmc': bmc,
        'port': 623,
    }


_configattributes = ('secret.hardwaremanagementuser',
                     'secret.hardwaremanagementpassword',
                     'secret.ipmikg', 'hardwaremanagement.manager')


def _donothing(data):
    # a dummy function to avoid some awkward exceptions from
    # zombie pyghmi console objects
    pass


class IpmiConsole(conapi.Console):
    configattributes = frozenset(_configattributes)

    def __init__(self, node, config):
        self.error = None
        self.datacallback = None
        crypt = config.decrypt
        self.solconnection = None
        config.decrypt = True
        self.broken = False
        configdata = config.get_node_attributes([node], _configattributes)
        connparams = get_conn_params(node, configdata[node])
        config.decrypt = crypt
        self.username = connparams['username']
        self.password = connparams['<PASSWORD>phrase']
        self.kg = connparams['kg']
        self.bmc = connparams['bmc']
        self.port = connparams['port']
        self.connected = False
        # Cannot actually create console until 'connect', when we get callback

    def __del__(self):
        self.solconnection = None

    def handle_data(self, data):
        if type(data) == dict:
            if 'error' in data:
                self.solconnection = None
                self.broken = True
                self.error = data['error']
                if self.connected:
                    self.connected = False
                    self.datacallback(conapi.ConsoleEvent.Disconnect)
        else:
            self.datacallback(data)

    def connect(self, callback):
        self.datacallback = callback
        # we provide a weak reference to pyghmi as otherwise we'd
        # have a circular reference and reference counting would never get
        # out...
        try:
            self.solconnection = console.Console(bmc=self.bmc, port=self.port,
                                                 userid=self.username,
                                                 password=<PASSWORD>,
                                                 kg=self.kg, force=True,
                                                 iohandler=self.handle_data)
            self.solconnection.outputlock = NullLock()
            while (self.solconnection and not self.solconnection.connected and
                   not (self.broken or self.solconnection.broken or
                        self.solconnection.ipmi_session.broken)):
                w = eventlet.event.Event()
                _ipmiwaiters.append(w)
                w.wait(15)
            if (self.broken or not self.solconnection or
                    self.solconnection.broken or
                    self.solconnection.ipmi_session.broken):
                if not self.error:
                    self.error = 'Unknown error'
                if (self.error.startswith('Incorrect password') or
                        self.error.startswith('Unauthorized name')):
                    raise exc.TargetEndpointBadCredentials
                else:
                    raise exc.TargetEndpointUnreachable(self.error)
            self.connected = True
        except socket.gaierror as err:
            raise exc.TargetEndpointUnreachable(str(err))

    def close(self):
        if self.solconnection is not None:
            # break the circular reference here
            self.solconnection.out_handler = _donothing
            self.solconnection.close()
        self.solconnection = None
        self.datacallback = None
        self.broken = True
        self.error = "closed"

    def write(self, data):
        self.solconnection.send_data(data)

    def send_break(self):
        self.solconnection.send_break()


def perform_requests(operator, nodes, element, cfg, inputdata):
    cryptit = cfg.decrypt
    cfg.decrypt = True
    configdata = cfg.get_node_attributes(nodes, _configattributes)
    cfg.decrypt = cryptit
    resultdata = queue.LightQueue()
    livingthreads = set([])
    for node in nodes:
        livingthreads.add(_ipmiworkers.spawn(
            perform_request, operator, node, element, configdata, inputdata,
            cfg, resultdata))
    while livingthreads:
        try:
            datum = resultdata.get(timeout=10)
            while datum:
                if datum != 'Done':
                    if isinstance(datum, Exception):
                        raise datum
                    yield datum
                datum = resultdata.get_nowait()
        except queue.Empty:
            pass
        for t in list(livingthreads):
            if t.dead:
                livingthreads.discard(t)
    try:
        # drain queue if a thread put something on the queue and died
        while True:
            datum = resultdata.get_nowait()
            if datum != 'Done':
                yield datum
    except queue.Empty:
        pass


def perform_request(operator, node, element,
                    configdata, inputdata, cfg, results):
        try:
            return IpmiHandler(operator, node, element, configdata, inputdata,
                               cfg, results).handle_request()
        except pygexc.IpmiException as ipmiexc:
            excmsg = str(ipmiexc)
            if excmsg in ('Session no longer connected', 'timeout'):
                results.put(msg.ConfluentTargetTimeout(node))
            else:
                results.put(msg.ConfluentNodeError(node, excmsg))
                raise
        except exc.TargetEndpointUnreachable as tu:
            results.put(msg.ConfluentTargetTimeout(node, str(tu)))
        except ssl.SSLEOFError:
            results.put(msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC'))
        except exc.PubkeyInvalid:
            results.put(msg.ConfluentNodeError(
                node,
                'Mismatch detected between target certificate fingerprint '
                'and pubkeys.tls_hardwaremanager attribute'))
        except pygexc.InvalidParameterValue as e:
            results.put(msg.ConfluentNodeError(node, str(e)))
        except Exception as e:
            results.put(e)
            raise
        finally:
            results.put('Done')

persistent_ipmicmds = {}

class IpmiHandler(object):
    def __init__(self, operation, node, element, cfd, inputdata, cfg, output):
        self.sensormap = {}
        self.usedlabels = {}
        self.invmap = {}
        self.output = output
        self.sensorcategory = None
        self.broken = False
        self.error = None
        eventlet.sleep(0)
        self.cfg = cfd[node]
        self.current_user = cfg.current_user
        self.loggedin = False
        self.node = node
        self.element = element
        self.op = operation
        connparams = get_conn_params(node, self.cfg)
        self.ipmicmd = None
        self.inputdata = inputdata
        self.tenant = cfg.tenant
        tenant = cfg.tenant
        if ((node, tenant) not in persistent_ipmicmds or
                not persistent_ipmicmds[(node, tenant)].ipmi_session.logged or
                persistent_ipmicmds[(node, tenant)].ipmi_session.broken):
            try:
                persistent_ipmicmds[(node, tenant)].close_confluent()
            except KeyError:  # was no previous session
                pass
            try:
                persistent_ipmicmds[(node, tenant)] = IpmiCommandWrapper(
                    node, cfg, bmc=connparams['bmc'],
                    userid=connparams['username'],
                    password=connparams['<PASSWORD>'], kg=connparams['kg'],
                    port=connparams['port'], onlogon=self.logged)

                ipmisess = persistent_ipmicmds[(node, tenant)].ipmi_session
                begin = util.monotonic_time()
                while ((not (self.broken or self.loggedin)) and
                               (util.monotonic_time() - begin) < 30):
                    ipmisess.wait_for_rsp(31 - (util.monotonic_time() - begin))
                if not (self.broken or self.loggedin):
                    ipmisess._mark_broken()
                    raise exc.TargetEndpointUnreachable(
                        "Login process to " + connparams['bmc'] + " died")
            except socket.gaierror as ge:
                if ge[0] == -2:
                    raise exc.TargetEndpointUnreachable(ge[1])
                raise
        self.ipmicmd = persistent_ipmicmds[(node, tenant)]

    bootdevices = {
        'optical': 'cd'
    }

    def logged(self, response, ipmicmd):
        if 'error' in response:
            self.broken = True
            self.error = response['error']
        else:
            self.ipmicmd = ipmicmd
            self.loggedin = True

    def handle_request(self):
        if self.broken:
            if (self.error == 'timeout' or
                    'Insufficient resources' in self.error):
                self.error = self.error.replace(' reported in RAKP4', '')
                self.output.put(msg.ConfluentTargetTimeout(
                    self.node, self.error))
                return
            elif ('Unauthorized' in self.error or
                    'Incorrect password' in self.error):
                self.output.put(
                    msg.ConfluentTargetInvalidCredentials(self.node))
                return
            else:
                raise Exception(self.error)
        if self.element == ['power', 'state']:
            self.power()
        elif self.element == ['_enclosure', 'reseat_bay']:
            self.reseat_bay()
        elif self.element == ['boot', 'nextdevice']:
            self.bootdevice()
        elif self.element == ['health', 'hardware']:
            self.health()
        elif self.element == ['identify']:
            self.identify()
        elif self.element[0] == 'sensors':
            self.handle_sensors()
        elif self.element[0] == 'configuration':
            self.handle_configuration()
        elif self.element[:3] == ['inventory', 'firmware', 'updates']:
            self.handle_update()
        elif self.element[0] == 'inventory':
            self.handle_inventory()
        elif self.element == ['media', 'attach']:
            self.handle_attach_media()
        elif self.element == ['media', 'detach']:
            self.handle_detach_media()
        elif self.element == ['media', 'uploads']:
            self.handle_media_upload()
        elif self.element == ['media', 'current']:
            self.handle_list_media()
        elif self.element == ['events', 'hardware', 'log']:
            self.do_eventlog()
        elif self.element == ['events', 'hardware', 'decode']:
            self.decode_alert()
        elif self.element == ['console', 'license']:
            self.handle_license()
        elif self.element == ['support', 'servicedata']:
            self.handle_servicedata_fetch()
        elif self.element == ['description']:
            self.handle_description()
        else:
            raise Exception('Not Implemented')

    def handle_update(self):
        u = firmwaremanager.Updater(self.node, self.ipmicmd.update_firmware,
                                    self.inputdata.filename, self.tenant,
                                    bank=self.inputdata.bank)
        self.output.put(
            msg.CreatedResource(
                'nodes/{0}/inventory/firmware/updates/active/{1}'.format(
                    self.node, u.name)))

    def handle_media_upload(self):
        u = firmwaremanager.Updater(self.node, self.ipmicmd.upload_media,
                                     self.inputdata.filename, self.tenant,
                                     type='mediaupload')
        self.output.put(msg.CreatedResource(
            'nodes/{0}/media/uploads/{1}'.format(self.node, u.name)))

    def handle_servicedata_fetch(self):
        u = firmwaremanager.Updater(
            self.node, self.ipmicmd.get_diagnostic_data,
            self.inputdata.filename, self.tenant, type='ffdc',
            owner=self.current_user)
        self.output.put(msg.CreatedResource(
            'nodes/{0}/support/servicedata/{1}'.format(self.node, u.name)))

    def handle_attach_media(self):
        try:
            self.ipmicmd.attach_remote_media(self.inputdata.filename)
        except pygexc.UnsupportedFunctionality as uf:
            self.output.put(msg.ConfluentNodeError(self.node, str(uf)))

    def handle_detach_media(self):
        self.ipmicmd.detach_remote_media()

    def handle_list_media(self):
        for media in self.ipmicmd.list_media():
            self.output.put(msg.Media(self.node, media))

    def handle_configuration(self):
        if self.element[1:3] == ['management_controller', 'alerts']:
            return self.handle_alerts()
        elif self.element[1:3] == ['management_controller', 'users']:
            return self.handle_users()
        elif self.element[1:3] == ['management_controller', 'net_interfaces']:
            return self.handle_nets()
        elif self.element[1:3] == ['management_controller', 'reset']:
            return self.handle_reset()
        elif self.element[1:3] == ['management_controller', 'identifier']:
            return self.handle_identifier()
        elif self.element[1:3] == ['management_controller', 'hostname']:
            return self.handle_hostname()
        elif self.element[1:3] == ['management_controller', 'domain_name']:
            return self.handle_domain_name()
        elif self.element[1:3] == ['management_controller', 'ntp']:
            return self.handle_ntp()
        elif self.element[1:3] == ['system', 'all']:
            return self.handle_sysconfig()
        elif self.element[1:3] == ['system', 'advanced']:
            return self.handle_sysconfig(True)
        elif self.element[1:3] == ['system', 'clear']:
            return self.handle_sysconfigclear()
        raise Exception('Not implemented')

    def decode_alert(self):
        inputdata = self.inputdata.get_alert(self.node)
        specifictrap = int(inputdata['.1.3.6.1.6.3.1.1.4.1.0'].rpartition(
            '.')[-1])
        for tmpvarbind in inputdata:
            if tmpvarbind.endswith('3183.1.1'):
                varbinddata = inputdata[tmpvarbind]
        varbinddata = hex2bin(varbinddata)
        event = self.ipmicmd.decode_pet(specifictrap, varbinddata)
        self.pyghmi_event_to_confluent(event)
        self.output.put(msg.EventCollection((event,), name=self.node))

    def handle_alerts(self):
        if self.element[3] == 'destinations':
            if len(self.element) == 4:
                # A list of destinations
                maxdest = self.ipmicmd.get_alert_destination_count()
                for alertidx in xrange(0, maxdest + 1):
                    self.output.put(msg.ChildCollection(alertidx))
                return
            elif len(self.element) == 5:
                alertidx = int(self.element[-1])
                if self.op == 'read':
                    destdata = self.ipmicmd.get_alert_destination(alertidx)
                    self.output.put(msg.AlertDestination(
                        ip=destdata['address'],
                        acknowledge=destdata['acknowledge_required'],
                        acknowledge_timeout=destdata.get('acknowledge_timeout', None),
                        retries=destdata['retries'],
                        name=self.node))
                    return
                elif self.op == 'update':
                    alertparms = self.inputdata.alert_params_by_node(
                        self.node)
                    alertargs = {}
                    if 'acknowledge' in alertparms:
                        alertargs['acknowledge_required'] = alertparms['acknowledge']
                    if 'acknowledge_timeout' in alertparms:
                        alertargs['acknowledge_timeout'] = alertparms['acknowledge_timeout']
                    if 'ip' in alertparms:
                        alertargs['ip'] = alertparms['ip']
                    if 'retries' in alertparms:
                        alertargs['retries'] = alertparms['retries']
                    self.ipmicmd.set_alert_destination(destination=alertidx,
                                                       **alertargs)
                    return
                elif self.op == 'delete':
                    self.ipmicmd.clear_alert_destination(alertidx)
                    return
        raise Exception('Not implemented')

    def handle_nets(self):
        if len(self.element) == 3:
            if self.op != 'read':
                self.output.put(
                    msg.ConfluentNodeError(self.node, 'Unsupported operation'))
                return
            self.output.put(msg.ChildCollection('management'))
        elif len(self.element) == 4 and self.element[-1] == 'management':
            if self.op == 'read':
                lancfg = self.ipmicmd.get_net_configuration()
                self.output.put(msg.NetworkConfiguration(
                    self.node, ipv4addr=lancfg['ipv4_address'],
                    ipv4gateway=lancfg['ipv4_gateway'],
                    ipv4cfgmethod=lancfg['ipv4_configuration'],
                    hwaddr=lancfg['mac_address']
                ))
            elif self.op == 'update':
                config = self.inputdata.netconfig(self.node)
                try:
                    self.ipmicmd.set_net_configuration(
                        ipv4_address=config['ipv4_address'],
                        ipv4_configuration=config['ipv4_configuration'],
                        ipv4_gateway=config['ipv4_gateway'])
                except socket.error as se:
                    self.output.put(msg.ConfluentNodeError(self.node,
                                                           se.message))
                except ValueError as e:
                    if e.message == 'negative shift count':
                        self.output.put(msg.ConfluentNodeError(
                            self.node, 'Invalid prefix length given'))
                    else:
                        raise

    def handle_users(self):
        # Create user
        if len(self.element) == 3:
            if self.op == 'update':
                user = self.inputdata.credentials[self.node]
                self.ipmicmd.create_user(uid=user['uid'], name=user['username'],
                                    password=<PASSWORD>['password'],
                                    callback=True,link_auth=True, ipmi_msg=True,
                                    privilege_level=user['privilege_level'])
            # A list of users
            self.output.put(msg.ChildCollection('all'))
            for user in self.ipmicmd.get_users():
                self.output.put(msg.ChildCollection(user, candelete=True))
            return
        # List all users
        elif len(self.element) == 4 and self.element[-1] == 'all':
            users = []
            for user in self.ipmicmd.get_users():
                users.append(self.ipmicmd.get_user(uid=user))
            self.output.put(msg.UserCollection(users=users, name=self.node))
            return
        # Update user
        elif len(self.element) == 4:
            user = int(self.element[-1])
            if self.op == 'read':
                data = self.ipmicmd.get_user(uid=user)
                self.output.put(msg.User(
                    uid=data['uid'],
                    username=data['name'],
                    privilege_level=data['access']['privilege_level'],
                    name=self.node))
                return
            elif self.op == 'update':
                user = self.inputdata.credentials[self.node]

                if 'username' in user:
                    self.ipmicmd.set_user_name(uid=user['uid'],
                                               name=user['username'])
                if 'privilege_level' in user:
                    self.ipmicmd.set_user_access(uid=user['uid'],
                                    privilege_level=user['privilege_level'])
                if 'password' in user:
                    self.ipmicmd.set_user_password(uid=user['uid'],
                                                   password=user['password'])
                    self.ipmicmd.set_user_password(uid=user['uid'],
                                    mode='enable', password=user['password'])
                if 'enabled' in user:
                    if user['enabled'] == 'yes':
                        mode = 'enable'
                    else:
                        mode = 'disable'
                    self.ipmicmd.disable_user(user['uid'], mode)
                return
            elif self.op == 'delete':
                self.ipmicmd.user_delete(uid=user)
                return

    def do_eventlog(self):
        eventout = []
        clear = False
        if self.op == 'delete':
            clear = True
        for event in self.ipmicmd.get_event_log(clear):
            self.pyghmi_event_to_confluent(event)
            eventout.append(event)
        self.output.put(msg.EventCollection(eventout, name=self.node))

    def pyghmi_event_to_confluent(self, event):
        event['severity'] = _str_health(event.get('severity', 'unknown'))
        if 'event_data' in event:
            event['event'] = '{0} - {1}'.format(
                event['event'], event['event_data'])
        if 'event_id' in event:
            event['id'] = '{0}.{1}'.format(event['event_id'],
                                           event['component_type_id'])

    def make_inventory_map(self):
        invnames = self.ipmicmd.get_inventory_descriptions()
        for name in invnames:
            self.invmap[simplify_name(name)] = name

    def make_sensor_map(self, sensors=None):
        if sensors is None:
            sensors = self.ipmicmd.get_sensor_descriptions()
        for sensor in sensors:
            resourcename = sensor['name']
            self.sensormap[simplify_name(resourcename)] = resourcename

    def read_sensors(self, sensorname):
        if sensorname == 'all':
            sensors = self.ipmicmd.get_sensor_descriptions()
            readings = []
            for sensor in filter(self.match_sensor, sensors):
                try:
                    reading = self.ipmicmd.get_sensor_reading(
                        sensor['name'])
                except pygexc.IpmiException as ie:
                    if ie.ipmicode == 203:
                        self.output.put(msg.SensorReadings([EmptySensor(
                            sensor['name'])], name=self.node))
                        continue
                    raise
                if hasattr(reading, 'health'):
                    reading.health = _str_health(reading.health)
                readings.append(reading)
            self.output.put(msg.SensorReadings(readings, name=self.node))
        else:
            self.make_sensor_map()
            if sensorname not in self.sensormap:
                self.output.put(
                    msg.ConfluentTargetNotFound(self.node,
                                                'Sensor not found'))
                return
            try:
                reading = self.ipmicmd.get_sensor_reading(
                    self.sensormap[sensorname])
                if hasattr(reading, 'health'):
                    reading.health = _str_health(reading.health)
                self.output.put(
                    msg.SensorReadings([reading],
                                       name=self.node))
            except pygexc.IpmiException as ie:
                if ie.ipmicode == 203:
                    self.output.put(msg.ConfluentResourceUnavailable(
                        self.node, 'Unavailable'
                    ))
                else:
                    self.output.put(msg.ConfluentTargetTimeout(self.node))

    def list_inventory(self):
        try:
            components = self.ipmicmd.get_inventory_descriptions()
        except pygexc.IpmiException:
            self.output.put(msg.ConfluentTargetTimeout(self.node))
            return
        self.output.put(msg.ChildCollection('all'))
        for component in components:
            self.output.put(msg.ChildCollection(simplify_name(component)))

    def list_firmware(self):
        self.output.put(msg.ChildCollection('all'))
        for id, data in self.ipmicmd.get_firmware():
            self.output.put(msg.ChildCollection(simplify_name(id)))

    def read_firmware(self, component):
        items = []
        errorneeded = False
        try:
            complist = () if component == 'all' else (component,)
            for id, data in self.ipmicmd.get_firmware(complist):
                if (component in ('core', 'all') or
                        component == simplify_name(id)):
                    items.append({id: data})
        except ssl.SSLEOFError:
            errorneeded = msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC while trying to read extended '
                           'information')
        except exc.PubkeyInvalid:
            errorneeded = msg.ConfluentNodeError(
                self.node,
                'Extended information unavailable, mismatch detected between '
                'target certificate fingerprint and '
                'pubkeys.tls_hardwaremanager attribute')
        self.output.put(msg.Firmware(items, self.node))
        if errorneeded:
            self.output.put(errorneeded)

    def handle_inventory(self):
        if self.element[1] == 'firmware':
            if len(self.element) == 3:
                return self.list_firmware()
            elif len(self.element) == 4:
                return self.read_firmware(self.element[-1])
        elif self.element[1] == 'hardware':
            if len(self.element) == 3:  # list things in inventory
                return self.list_inventory()
            elif len(self.element) == 4:  # actually read inventory data
                return self.read_inventory(self.element[-1])
        raise Exception('Unsupported scenario...')

    def list_leds(self):
        self.output.put(msg.ChildCollection('all'))
        for category, info in self.ipmicmd.get_leds():
            self.output.put(msg.ChildCollection(simplify_name(category)))

    def read_leds(self, component):
        led_categories = []
        for category, info in self.ipmicmd.get_leds():
            if component == 'all' or component == simplify_name(category):
                led_categories.append({category: info})
        self.output.put(msg.LEDStatus(led_categories, self.node))

    def read_inventory(self, component):
        errorneeded = False
        try:
            invitems = []
            if component == 'all':
                for invdata in self.ipmicmd.get_inventory():
                    if invdata[1] is None:
                        newinf = {'present': False, 'information': None}
                    else:
                        sanitize_invdata(invdata[1])
                        newinf = {'present': True, 'information': invdata[1]}
                    newinf['name'] = invdata[0]
                    self.add_invitem(invitems, newinf)
            else:
                self.make_inventory_map()
                compname = self.invmap.get(component, None)
                if compname is None:
                    self.output.put(msg.ConfluentTargetNotFound())
                    return
                invdata = self.ipmicmd.get_inventory_of_component(compname)
                if invdata is None:
                    newinf = {'present': False, 'information': None}
                else:
                    sanitize_invdata(invdata)
                    newinf = {'present': True, 'information': invdata}
                newinf['name'] = compname
                self.add_invitem(invitems, newinf)
        except ssl.SSLEOFError:
            errorneeded = msg.ConfluentNodeError(
                self.node, 'Unable to communicate with the https server on '
                           'the target BMC while trying to read extended '
                           'information')
        except exc.PubkeyInvalid:
            errorneeded = msg.ConfluentNodeError(
                self.node,
                'Extended information unavailable, mismatch detected between '
                'target certificate fingerprint and '
                'pubkeys.tls_hardwaremanager attribute')
        newinvdata = {'inventory': invitems}
        self.output.put(msg.KeyValueData(newinvdata, self.node))
        if errorneeded:
            self.output.put(errorneeded)

    def add_invitem(self, invitems, newinf):
        if fnmatch(newinf['name'], 'Adapter ??:??:??') or fnmatch(
                newinf['name'], 'PCIeGen? x*'):
            myinf = newinf.get('information', {})
            sdid = myinf.get('PCI Subsystem Device ID', None)
            svid = myinf.get('PCI Subsystem Vendor ID', None)
            did = myinf.get('PCI Device ID', None)
            vid = myinf.get('PCI Vendor ID', None)
            vstr, dstr = get_pci_text_from_ids(sdid, svid, did, vid)
            if vstr:
                newinf['information']['PCI Vendor'] = vstr
            if dstr:
                newname = dstr
                instance = 1
                while newname in self.usednames:
                    instance += 1
                    newname = dstr + ' {1}'.format(instance)
                newinf['name'] = newname
        invitems.append(newinf)

    def handle_sensors(self):
        if self.element[-1] == '':
            self.element = self.element[:-1]
        if len(self.element) < 3:
            return
        self.sensorcategory = self.element[2]
        # list sensors per category
        if len(self.element) == 3 and self.element[-2] == 'hardware':
            if self.sensorcategory == 'leds':
                return self.list_leds()
            return self.list_sensors()
        elif len(self.element) == 4:  # resource requested
            if self.sensorcategory == 'leds':
                return self.read_leds(self.element[-1])
            return self.read_sensors(self.element[-1])

    def match_sensor(self, sensor):
        if self.sensorcategory == 'all':
            return True
{'fraction_non_alphanumeric': 0.10230426813301911, 'fraction_numerical': 0.004490704372872479, 'mean_word_length': 1.826308973172988, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2494999', 'n_tokens_mistral': 21249, 'n_tokens_neox': 20395, 'n_words': 4501}