<commit_before># -*- encoding: utf-8 -*-
'''
generic data to splunk returner

Deliver generic HubbleStack event data into Splunk using the HTTP
event collector. Required config/pillar settings:

.. code-block:: yaml

    hubblestack:
      returner:
        generic:
          - token: <KEY>
            indexer: splunk-indexer.domain.tld
            index: hubble
            sourcetype_pulsar: generic
'''

import time
from hubblestack.hec import http_event_collector, get_splunk_options

def _get_key(dat, k, d=None):
    if isinstance(dat, dict):
        return dat.pop(k, d)
    return d

def returner(retdata):
    try:
        retdata = retdata['return']
    except KeyError:
        return

    opts_list = get_splunk_options()
    for opts in opts_list:
        http_event_collector_key = opts['token']
        http_event_collector_host = opts['indexer']
        http_event_collector_port = opts['port']
        hec_ssl = opts['http_event_server_ssl']
        proxy = opts['proxy']
        timeout = opts['timeout']
        http_event_collector_ssl_verify = opts['http_event_collector_ssl_verify']

        hec = http_event_collector(http_event_collector_key, http_event_collector_host,
                                   http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl,
                                   http_event_collector_ssl_verify=http_event_collector_ssl_verify,
                                   proxy=proxy, timeout=timeout)

        t_sourcetype = _get_key(retdata, 'sourcetype', 'hubble_generic')
        t_time       = _get_key(retdata, 'time', time.time())
        events       = _get_key(retdata, 'event', _get_key(retdata, 'events'))

        if events is None:
            return

        if not isinstance(events, (list,tuple)):
            events = [events]

        if len(events) < 1 or (len(events) == 1 and events[0] is None):
            return

        for event in events:
            payload = {'host': __grains__.get('fqdn', __grains__.get('id')), 'event': event,
                'sourcetype': _get_key(event, 'sourcetype', t_sourcetype),
                'time': str(int(_get_key(event, 'time', t_time)))}
            hec.batchEvent(payload)
        hec.flushBatch()
<commit_msg>The index was missing from the payload.<commit_after># -*- encoding: utf-8 -*-
'''
generic data to splunk returner

Deliver generic HubbleStack event data into Splunk using the HTTP
event collector. Required config/pillar settings:

.. code-block:: yaml

    hubblestack:
      returner:
        generic:
          - token: <KEY>
            indexer: splunk-indexer.domain.tld
            index: hubble
            sourcetype_pulsar: generic
'''

import time
from hubblestack.hec import http_event_collector, get_splunk_options

def _get_key(dat, k, d=None):
    if isinstance(dat, dict):
        return dat.pop(k, d)
    return d

def returner(retdata):
    try:
        retdata = retdata['return']
    except KeyError:
        return

    opts_list = get_splunk_options()
    for opts in opts_list:
        http_event_collector_key = opts['token']
        http_event_collector_host = opts['indexer']
        http_event_collector_port = opts['port']
        hec_ssl = opts['http_event_server_ssl']
        proxy = opts['proxy']
        timeout = opts['timeout']
        http_event_collector_ssl_verify = opts['http_event_collector_ssl_verify']

        hec = http_event_collector(http_event_collector_key, http_event_collector_host,
                                   http_event_port=http_event_collector_port, http_event_server_ssl=hec_ssl,
                                   http_event_collector_ssl_verify=http_event_collector_ssl_verify,
                                   proxy=proxy, timeout=timeout)

        t_sourcetype = _get_key(retdata, 'sourcetype', 'hubble_generic')
        t_time       = _get_key(retdata, 'time', time.time())
        events       = _get_key(retdata, 'event', _get_key(retdata, 'events'))

        if events is None:
            return

        if not isinstance(events, (list,tuple)):
            events = [events]

        if len(events) < 1 or (len(events) == 1 and events[0] is None):
            return

        idx = opts.get('index')

        for event in events:
            payload = {
                'host': __grains__.get('fqdn', __grains__.get('id')),
                'event': event,
                'sourcetype': _get_key(event, 'sourcetype', t_sourcetype),
                'time': str(int(_get_key(event, 'time', t_time)))}
            if idx:
                payload['index'] = idx
            hec.batchEvent(payload)
        hec.flushBatch()

{'fraction_non_alphanumeric': 0.0994606256742179, 'fraction_numerical': 0.001725997842502697, 'mean_word_length': 2.3328540618260245, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3951240', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1404, 'n_words': 304}