<commit_before># -*- coding: utf-8 -*-
"""
Parses simple newline separated list of IPs.

Docs:
 - https://feodotracker.abuse.ch/blocklist/
 - https://zeustracker.abuse.ch/blocklist.php
"""
import re

import dateutil

from intelmq.lib.bot import ParserBot
from intelmq.lib import utils
from intelmq.lib.exceptions import PipelineError


FEEDS = {
    'https://feodotracker.abuse.ch/downloads/ipblocklist.csv': {
        'format': [
            'time.source',
            'source.ip',
            'source.port',
            'malware.name'
        ],
        'malware': 'Cridex'
    },
    'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist': {
        'format': [
            'source.ip'
        ],
        'malware': 'Zeus'
    },
    'https://zeustracker.abuse.ch/blocklist.php?download=badips': {
        'format': [
            'source.ip'
        ],
        'malware': 'Zeus'
    }
}

SOURCE_FEEDS = {'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist': 'Cridex',
                'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist': 'Zeus',
                'https://zeustracker.abuse.ch/blocklist.php?download=badips': 'Zeus'}


class AbusechIPParserBot(ParserBot):
    lastgenerated = None

    def parse(self, report: dict):
        lines = utils.base64_decode(report.get("raw")).splitlines()
        regex = re.compile(r'^#+.*')  # filter out comments
        top_line = list(filter(lambda i: not regex.search(i), lines))[0]
        feed = report['feed.url']

        fields = top_line.split(',')
        if len(fields) is not len(FEEDS[feed]['format']):
            self.logger.warning(
                f"Feed '{feed}' has not the expected length of fields: {len(fields)} != {len(FEEDS[feed]['format'])}")
            raise PipelineError("Abusech ip parser bad config")

        for line in utils.base64_decode(report.get("raw")).splitlines():
            line = line.strip()
            if not any([line.startswith(prefix) for prefix in self.ignore_lines_starting]):
                yield line

    def parse_line(self, line, report):<commit_msg>Remove unused elements<commit_after># -*- coding: utf-8 -*-
"""
Parses simple newline separated list of IPs.

Docs:
 - https://feodotracker.abuse.ch/blocklist/
 - https://zeustracker.abuse.ch/blocklist.php
"""
import re

import dateutil

from intelmq.lib.bot import ParserBot
from intelmq.lib import utils
from intelmq.lib.exceptions import PipelineError


FEEDS = {
    'https://feodotracker.abuse.ch/downloads/ipblocklist.csv': {
        'format': [
            'time.source',
            'source.ip',
            'source.port',
            'malware.name'
        ],
        'malware': 'Cridex'
    },
    'https://zeustracker.abuse.ch/blocklist.php?download=ipblocklist': {
        'format': [
            'source.ip'
        ],
        'malware': 'Zeus'
    },
    'https://zeustracker.abuse.ch/blocklist.php?download=badips': {
        'format': [
            'source.ip'
        ],
        'malware': 'Zeus'
    }
}


class AbusechIPParserBot(ParserBot):
    lastgenerated = None

    def parse(self, report: dict):
        lines = utils.base64_decode(report.get("raw")).splitlines()
        regex = re.compile(r'^#+.*')  # filter out comments
        top_line = list(filter(lambda i: not regex.search(i), lines))[0]
        feed = report['feed.url']

        fields = top_line.split(',')
        if len(fields) is not len(FEEDS[feed]['format']):
            self.logger.warning(
                f"Feed '{feed}' has not the expected length of fields: {len(fields)} != {len(FEEDS[feed]['format'])}")
            raise PipelineError("Abusech ip parser bad config")

        for line in utils.base64_decode(report.get("raw")).splitlines():
            line = line.strip()
            if not any([line.startswith(prefix) for prefix in self.ignore_lines_starting]):
                yield line

    def parse_line(self, line, report):
{'fraction_non_alphanumeric': 0.149949083503055, 'fraction_numerical': 0.003054989816700611, 'mean_word_length': 2.4404553415061296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 13, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6258975', 'n_tokens_mistral': 1280, 'n_tokens_neox': 1248, 'n_words': 238}