<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import sys
from urlparse import urlparse
from intelmq.lib.bot import Bot


class Url2fqdnExpertBot(Bot):

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        for key in ["source.", "destination."]:

            key_url = key + "url"
            key_fqdn = key + "fqdn"
            if not event.contains(key_url):
                continue

            url = event.value(key_url)
            event.add(key_fqdn, urlparse(url).hostname, sanitize=True,
                      force=True)

        self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = Url2fqdnExpertBot(sys.argv[1])
    bot.start()<commit_msg>fixed issue with IP and python3<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import sys
import re
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse
from intelmq.lib.bot import Bot


class Url2fqdnExpertBot(Bot):

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        for key in ["source.", "destination."]:

            key_url = key + "url"
            key_fqdn = key + "fqdn"
            if not event.contains(key_url):
                continue

            hostname = urlparse(event.value(key_url)).hostname
            if not re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").match(hostname):
                event.add(key_fqdn, hostname, sanitize=True,
                          force=True)

        self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = Url2fqdnExpertBot(sys.argv[1])
    bot.start()
{'fraction_non_alphanumeric': 0.11039657020364416, 'fraction_numerical': 0.009110396570203644, 'mean_word_length': 2.1064891846921796, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '117058', 'n_tokens_mistral': 599, 'n_tokens_neox': 575, 'n_words': 132}