<commit_before># -*- coding: utf-8 -*-
"""
See README for database download.

TOOD: IPv6
"""
from __future__ import unicode_literals
import sys

from intelmq.lib.bot import Bot


class TorExpertBot(Bot):

    database = list()

    def init(self):
        self.logger.info("Loading TOR exit node IPs.")

        try:
            with open(self.parameters.database) as fp:
                for line in fp:
                    line = line.strip()

                    if len(line) == 0 or line[0] == "#":
                        continue

                    ip_list = line.split("[")[1]
                    ip_list = ip_list.split("]")[0]
                    ip_list = ip_list.split(",")

                    for ip in ip_list:
                        TorExpertBot.database.append(ip.strip())

        except IOError:
            self.logger.critical("TOR rule not defined or failed on open.")
            self.stop()

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        keys = ['source.%s', 'destination.%s']

        for key in keys:
            if event.contains(key % 'ip'):
                if event.value(key % 'ip') in TorExpertBot.database:
                    event.add(key % 'tor_node', True)

        self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = TorExpertBot(sys.argv[1])
    bot.start()
<commit_msg>ENH: tor expert: simplify code and use sets<commit_after># -*- coding: utf-8 -*-
"""
See README for database download.

TOOD: IPv6
"""
from __future__ import unicode_literals
import sys

from intelmq.lib.bot import Bot


class TorExpertBot(Bot):

    database = set()

    def init(self):
        self.logger.info("Loading TOR exit node IPs.")

        try:
            with open(self.parameters.database) as fp:
                for line in fp:
                    line = line.strip()

                    if len(line) == 0 or line[0] == "#":
                        continue

                    ip_list = line[line.find("[")+1:line.find("]")].split(",")

                    for ip in ip_list:
                        self.database.add(ip.strip())

        except IOError:
            self.logger.critical("TOR rule not defined or failed on open.")
            self.stop()

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        for key in ["source.", "destination."]:
            if event.contains(key + 'ip'):
                if event.value(key + 'ip') in self.database:
                    event.add(key + 'tor_node', True)

        self.send_message(event)
        self.acknowledge_message()


if __name__ == "__main__":
    bot = TorExpertBot(sys.argv[1])
    bot.start()

{'fraction_non_alphanumeric': 0.1191145467322558, 'fraction_numerical': 0.004567814476458187, 'mean_word_length': 1.645910780669145, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5923663', 'n_tokens_mistral': 852, 'n_tokens_neox': 791, 'n_words': 213}