<commit_before># -*- coding: utf-8 -*-
"""A collector for grabbing appropriately tagged events from MISP.

Parameters:
  - misp_url: URL of the MISP server
  - misp_key: API key for accessing MISP
  - misp_tag_to_process: MISP tag identifying events to be processed
  - misp_tag_processed: MISP tag identifying events that have been processed


PyMISP versions released after January 2020 will no longer support the
"old" PyMISP class.
For compatibility:
 * older versions of pymisp still work with this bot
 * the deprecated parameter `misp_verify` will create a DeprecationWarning
"""
import json
import warnings
import sys

from intelmq.lib.bot import CollectorBot
from intelmq.lib.exceptions import MissingDependencyError

try:
    if sys.version_info >= (3, 6):
        try:
            from pymisp import ExpandedPyMISP as PyMISP
        except ImportError:
            from pymisp import PyMISP
    else:
        from pymisp import PyMISP

except ImportError:
    PyMISP = None
    import_fail_reason = 'import'
except SyntaxError:
    PyMISP = None
    import_fail_reason = 'syntax'


class MISPCollectorBot(CollectorBot):

    def init(self):
        if PyMISP is None and import_fail_reason == 'syntax':
            raise MissingDependencyError("pymisp",
                                         version='>=2.4.36,<=172.16.58.3',
                                         additional_text="Python versions below 3.6 are "
                                                         "only supported by pymisp <= 172.16.58.3.")
        elif PyMISP is None:
            raise MissingDependencyError("pymisp")

        if hasattr(self.parameters, 'misp_verify'):
            self.parameters.http_verify_cert = self.parameters.misp_verify
            warnings.warn("The parameter 'misp_verify' is deprecated in favor of"
                          "'http_verify_cert'.", DeprecationWarning)

        # Initialize MISP connection
        self.misp = PyMISP(self.parameters.misp_url,
                           self.parameters.misp_key,
                           self.parameters.http_verify_cert)

    def process(self):
        # Grab the events from MISP
        misp_result = self.misp.search(
            tags=self.parameters.misp_tag_to_process
        )

        # Process the response and events

        # Compatibility with old pymisp versions before 2019:
        if 'response' in misp_result:
            misp_result = misp_result['response']

        # Extract the MISP event details
        for e in misp_result:
            misp_event = e['Event']

            # Send the results to the parser
            report = self.new_report()
            report.add('raw', json.dumps(misp_event, sort_keys=True))
            report.add('feed.url', self.parameters.misp_url)
            self.send_message(report)

        # Finally, update the tags on the MISP events.

        for misp_event in misp_result:
            if hasattr(self.parameters, 'misp_tag_processed'):
                # Add a 'processed' tag to the event
                self.misp.tag(misp_event['uuid'],
                              self.parameters.misp_tag_processed)

            # Remove the 'to be processed' tag
            self.misp.untag(misp_event['uuid'],
                            self.parameters.misp_tag_to_process)


BOT = MISPCollectorBot<commit_msg>add missing misp event key<commit_after># -*- coding: utf-8 -*-
"""A collector for grabbing appropriately tagged events from MISP.

Parameters:
  - misp_url: URL of the MISP server
  - misp_key: API key for accessing MISP
  - misp_tag_to_process: MISP tag identifying events to be processed
  - misp_tag_processed: MISP tag identifying events that have been processed


PyMISP versions released after January 2020 will no longer support the
"old" PyMISP class.
For compatibility:
 * older versions of pymisp still work with this bot
 * the deprecated parameter `misp_verify` will create a DeprecationWarning
"""
import json
import warnings
import sys

from intelmq.lib.bot import CollectorBot
from intelmq.lib.exceptions import MissingDependencyError

try:
    if sys.version_info >= (3, 6):
        try:
            from pymisp import ExpandedPyMISP as PyMISP
        except ImportError:
            from pymisp import PyMISP
    else:
        from pymisp import PyMISP

except ImportError:
    PyMISP = None
    import_fail_reason = 'import'
except SyntaxError:
    PyMISP = None
    import_fail_reason = 'syntax'


class MISPCollectorBot(CollectorBot):

    def init(self):
        if PyMISP is None and import_fail_reason == 'syntax':
            raise MissingDependencyError("pymisp",
                                         version='>=2.4.36,<=2.4.119.1',
                                         additional_text="Python versions below 3.6 are "
                                                         "only supported by pymisp <= 2.4.119.1.")
        elif PyMISP is None:
            raise MissingDependencyError("pymisp")

        if hasattr(self.parameters, 'misp_verify'):
            self.parameters.http_verify_cert = self.parameters.misp_verify
            warnings.warn("The parameter 'misp_verify' is deprecated in favor of"
                          "'http_verify_cert'.", DeprecationWarning)

        # Initialize MISP connection
        self.misp = PyMISP(self.parameters.misp_url,
                           self.parameters.misp_key,
                           self.parameters.http_verify_cert)

    def process(self):
        # Grab the events from MISP
        misp_result = self.misp.search(
            tags=self.parameters.misp_tag_to_process
        )

        # Process the response and events

        # Compatibility with old pymisp versions before 2019:
        if 'response' in misp_result:
            misp_result = misp_result['response']

        # Extract the MISP event details
        for e in misp_result:
            misp_event = e['Event']

            # Send the results to the parser
            report = self.new_report()
            report.add('raw', json.dumps(misp_event, sort_keys=True))
            report.add('feed.url', self.parameters.misp_url)
            self.send_message(report)

        # Finally, update the tags on the MISP events.

        for e in misp_result:
            misp_event = e['Event']
            if hasattr(self.parameters, 'misp_tag_processed'):
                # Add a 'processed' tag to the event
                self.misp.tag(misp_event['uuid'],
                              self.parameters.misp_tag_processed)

            # Remove the 'to be processed' tag
            self.misp.untag(misp_event['uuid'],
                            self.parameters.misp_tag_to_process)


BOT = MISPCollectorBot
{'fraction_non_alphanumeric': 0.07342865653857611, 'fraction_numerical': 0.009234435507893953, 'mean_word_length': 2.0859375, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6803258', 'n_tokens_mistral': 1952, 'n_tokens_neox': 1805, 'n_words': 531}