<commit_before>import urlparse
from intelmq.lib import utils
from intelmq.lib.bot import Bot, sys
from intelmq.lib.message import Event
from intelmq.lib.harmonization import DateTime, IPAddress


class VXVaultParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if not report.contains("raw"):
            self.acknowledge_message()

        raw_report = utils.base64_decode(report.value("raw"))
        for row in raw_report.split('\n'):

            row = row.strip()

            if len(row) == 0 or not row.startswith('http'):
                continue

            url_object = urlparse.urlparse(row)

            if not url_object:
                continue

            url      = url_object.geturl()
            hostname = url_object.hostname
            port     = url_object.port

            event = Event()

            if IPAddress.is_valid(hostname):
                event.add("source.ip", hostname, sanitize=True)
            else:
                event.add("source.fqdn", hostname, sanitize=True)

            time_observation = DateTime().generate_datetime_now()
            event.add('time.observation', time_observation, sanitize=True)
            event.add('feed.name', report.value("feed.name"))
            event.add('feed.url', report.value("feed.url"))
            event.add('classification.type', u'malware')
            event.add("source.url", url, sanitize=True)
            event.add("source.port", str(port), sanitize=True)
            event.add("raw", row, sanitize=True)

            self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = VXVaultParserBot(sys.argv[1])<commit_msg>BUG: Fix merge error in vxvault parser bot<commit_after>import urlparse
from intelmq.lib import utils
from intelmq.lib.bot import Bot, sys
from intelmq.lib.message import Event
from intelmq.lib.harmonization import DateTime, IPAddress


class VXVaultParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if not report.contains("raw"):
            self.acknowledge_message()

        raw_report = utils.base64_decode(report.value("raw"))
        for row in raw_report.split('\n'):

            row = row.strip()

            if len(row) == 0 or not row.startswith('http'):
                continue

            url_object = urlparse.urlparse(row)

            if not url_object:
                continue

            url      = url_object.geturl()
            hostname = url_object.hostname
            port     = url_object.port

            event = Event()

            if IPAddress.is_valid(hostname, sanitize=True):
                event.add("source.ip", hostname, sanitize=True)
            else:
                event.add("source.fqdn", hostname, sanitize=True)

            time_observation = DateTime().generate_datetime_now()
            event.add('time.observation', time_observation, sanitize=True)
            event.add('feed.name', report.value("feed.name"))
            event.add('feed.url', report.value("feed.url"))
            event.add('classification.type', u'malware')
            event.add("source.url", url, sanitize=True)
            event.add("source.port", str(port), sanitize=True)
            event.add("raw", row, sanitize=True)

            self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = VXVaultParserBot(sys.argv[1])
{'fraction_non_alphanumeric': 0.11491228070175438, 'fraction_numerical': 0.0023391812865497076, 'mean_word_length': 2.2061855670103094, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5921904', 'n_tokens_mistral': 1001, 'n_tokens_neox': 962, 'n_words': 197}