<commit_before>import logging
from datetime import timedelta

from core.feed import Feed
from core.observables import Ip
from core.errors import ObservableValidationError


class TorExitNodes(Feed):

    default_values = {
        "frequency": timedelta(hours=1),
        "name": "TorExitNodes",
        "source": "https://www.dan.me.uk/tornodes",
        "description": "Tor exit nodes",
    }

    def update(self):
        feed = self._make_requests().text

        start = feed.find('<!-- __BEGIN_TOR_NODE_LIST__ //-->') + len(
            '<!-- __BEGIN_TOR_NODE_LIST__ //-->')
        end = feed.find('<!-- __END_TOR_NODE_LIST__ //-->')

        feed_raw = feed[start:end].replace(
            '\n', '').replace('<br />', '\n').replace('&gt;', '>').replace(
                '&lt;', '<')

        feed = feed_raw.split('\n')
        if len(feed) > 10:
            self.status = "OK"

        feed = self._temp_feed_data_compare(feed_raw)

        for line in feed:
            self.analyze(line)
        return True

    def analyze(self, line):

        fields = line.split('|')

        if len(fields) < 8:
            return

        context = {}
        ip = fields[0]
        context['name'] = fields[1]
        context['router-port'] = fields[2]
        context['directory-port'] = fields[3]
        context['flags'] = fields[4]
        context['version'] = fields[6]
        context['contactinfo'] = fields[7]

        context['description'] = "Tor exit node: %s (%s)" % (
            context['name'], ip)
        context['source'] = self.name
        try:
            ip = Ip.get_or_create(value=fields[0])
            ip.add_context(context)
            ip.add_source(self.name)
            ip.tag(['tor'])
        except ObservableValidationError as e:
            logging.error(e)
<commit_msg>clenaup<commit_after>import logging
from datetime import timedelta

from core.feed import Feed
from core.observables import Ip
from core.errors import ObservableValidationError


class TorExitNodes(Feed):

    default_values = {
        "frequency": timedelta(hours=1),
        "name": "TorExitNodes",
        "source": "https://www.dan.me.uk/tornodes",
        "description": "Tor exit nodes",
    }

    def update(self):
        feed = self._make_requests().text

        start = feed.find('<!-- __BEGIN_TOR_NODE_LIST__ //-->') + len(
            '<!-- __BEGIN_TOR_NODE_LIST__ //-->')
        end = feed.find('<!-- __END_TOR_NODE_LIST__ //-->')

        feed_raw = feed[start:end].replace(
            '\n', '').replace('<br />', '\n').replace('&gt;', '>').replace(
                '&lt;', '<')

        feed = feed_raw.split('\n')
        if len(feed) > 10:
            self.status = "OK"

        feed = self._temp_feed_data_compare(feed_raw)

        for line in feed:
            self.analyze(line)

        return True

    def analyze(self, line):

        fields = line.split('|')

        if len(fields) < 8:
            return

        context = {
            'name': fields[1],
            'router-port': fields[2],
            'directory-port': fields[3],
            'flags': fields[4],
            'version': fields[6],
            'contactinfo': fields[7],
            'source': self.name,
            'description': "Tor exit node: %s (%s)" % (
                fields[1], fields[0]),
        }

        try:
            ip = Ip.get_or_create(value=fields[0])
            ip.add_context(context)
            ip.add_source(self.name)
            ip.tag(['tor'])
        except ObservableValidationError as e:
            logging.error(e)

{'fraction_non_alphanumeric': 0.1558113419427288, 'fraction_numerical': 0.007018528916339135, 'mean_word_length': 1.9349258649093903, 'pattern_counts': {'":': 8, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4137529', 'n_tokens_mistral': 1108, 'n_tokens_neox': 1056, 'n_words': 220}