<commit_before>import sys
import time
import traceback
import json

from splunklib.modularinput import Argument, Event, Scheme, Script

from amp4e_events_input_lib.amp_storage_wrapper import AmpStorageWrapper
from util.logger import logger
from util.stream_consumer import StreamConsumer


class Amp4eEventsInput(Script):
    # name: [description, required_on_create, required_on_edit]
    SCHEME_ARGUMENTS = {
        'stream_name': ['The event stream name', True, True],
        'event_types': ['Enter event type evt_ids for the stream', True, True],
        'groups': ['Enter group guids for the stream', False, False],
        'api_host': ['AMP for Endpoints API host', True, True],
        'api_id': ['3rd Party API Client ID provided by AMP for Endpoints', True, True],
        'api_key': ['API secret', True, True],
        'event_types_names': ['Event types names', True, True],
        'groups_names': ['Groups names', False, False]
    }

    def get_scheme(self):
        scheme = Scheme('Cisco AMP for Endpoints Events Input')
        scheme.description = 'Allows creating and managing event streams from AMP for Endpoints'
        scheme.use_external_validation = False
        scheme.use_single_instance = False
        self.__add_scheme_arguments(scheme)
        return scheme

    # DEPRECATED within app. Use only if necessary
    # https://docs.splunk.com/Documentation/SplunkCloud/6.6.0/AdvancedDev/ModInputsValidate
    def validate_input(self, validation_definition):
        pass

    # Runs once on splunk restart and then gets called every time new input is created
    # Checks if the stream needs to be deleted (i.e. user
    # deleted it) and performs the deletion via API (?).
    # Tries to set up the RabbitMQ connection with credentials from current stream.
    # If stream doesn't exist yet, exits.
    # Otherwise, fetches all events from queue and writes them to logs.
    def stream_events(self, inputs, ew):
        logger.info('In stream_events')
        for input_name, _ in inputs.inputs.items():
            logger.debug('Starting input ' + input_name)
            inputs.metadata['name'] = input_name.split('://', 1)[-1]
            stream = self.__stream_from_inputs(inputs)
            connection_data = stream.get('amqp_credentials')
            if connection_data is not None:
                consumer = StreamConsumer(connection_data,
                                          lambda event: self.__on_event_callback(event, ew,
                                                                                 {'input_name': input_name,
                                                                                  'host': stream.get('api_host'),
                                                                                  'index': stream.get('index')}))
                try:
                    consumer.run()
                    break  # break if we somehow have more than one input here
                except Exception as e:
                    logger.error(traceback.format_exc())
                    time.sleep(3)
                    raise e

    def __add_scheme_arguments(self, scheme):<commit_msg>remove extraneous logging statement<commit_after>import sys
import time
import traceback
import json

from splunklib.modularinput import Argument, Event, Scheme, Script

from amp4e_events_input_lib.amp_storage_wrapper import AmpStorageWrapper
from util.logger import logger
from util.stream_consumer import StreamConsumer


class Amp4eEventsInput(Script):
    # name: [description, required_on_create, required_on_edit]
    SCHEME_ARGUMENTS = {
        'stream_name': ['The event stream name', True, True],
        'event_types': ['Enter event type evt_ids for the stream', True, True],
        'groups': ['Enter group guids for the stream', False, False],
        'api_host': ['AMP for Endpoints API host', True, True],
        'api_id': ['3rd Party API Client ID provided by AMP for Endpoints', True, True],
        'api_key': ['API secret', True, True],
        'event_types_names': ['Event types names', True, True],
        'groups_names': ['Groups names', False, False]
    }

    def get_scheme(self):
        scheme = Scheme('Cisco AMP for Endpoints Events Input')
        scheme.description = 'Allows creating and managing event streams from AMP for Endpoints'
        scheme.use_external_validation = False
        scheme.use_single_instance = False
        self.__add_scheme_arguments(scheme)
        return scheme

    # DEPRECATED within app. Use only if necessary
    # https://docs.splunk.com/Documentation/SplunkCloud/6.6.0/AdvancedDev/ModInputsValidate
    def validate_input(self, validation_definition):
        pass

    # Runs once on splunk restart and then gets called every time new input is created
    # Checks if the stream needs to be deleted (i.e. user
    # deleted it) and performs the deletion via API (?).
    # Tries to set up the RabbitMQ connection with credentials from current stream.
    # If stream doesn't exist yet, exits.
    # Otherwise, fetches all events from queue and writes them to logs.
    def stream_events(self, inputs, ew):
        for input_name, _ in inputs.inputs.items():
            logger.debug('Starting input ' + input_name)
            inputs.metadata['name'] = input_name.split('://', 1)[-1]
            stream = self.__stream_from_inputs(inputs)
            connection_data = stream.get('amqp_credentials')
            if connection_data is not None:
                consumer = StreamConsumer(connection_data,
                                          lambda event: self.__on_event_callback(event, ew,
                                                                                 {'input_name': input_name,
                                                                                  'host': stream.get('api_host'),
                                                                                  'index': stream.get('index')}))
                try:
                    consumer.run()
                    break  # break if we somehow have more than one input here
                except Exception as e:
                    logger.error(traceback.format_exc())
                    time.sleep(3)
                    raise e

    def __add_scheme_arguments(self, scheme):
{'fraction_non_alphanumeric': 0.08235480799746113, 'fraction_numerical': 0.002856236115518883, 'mean_word_length': 2.0791402051783097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4520733', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1529, 'n_words': 518}