<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import datetime
import sys

from intelmq.lib import utils
from intelmq.lib.bot import Bot
from intelmq.lib.message import Event


def is_valid_date(strd):
    try:
        datetime.datetime.strptime(strd, '%Y%m%d')
        return True
    except Exception:
        return False


class MalwareDomainsParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report is None or not report.contains("raw"):
            self.acknowledge_message()
            return

        raw_report = utils.base64_decode(report.value("raw"))

        for row in raw_report.split('\n'):

            row = row.rstrip()

            if row.startswith("#") or len(row) == 0:
                continue

            values = row.split('\t')[1:]

            event = Event(report)

            event.add('source.fqdn', values[1], sanitize=True)
            event.add('event_description.text', values[2], sanitize=True)

            for i in range(4, len(values)):
                if is_valid_date(values[i]):
                    event.add('time.source',  # TODO: verify timezone
                              values[i] + "T00:00:00+00:00", sanitize=True)
                    break
<commit_msg>force adding time.source<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import datetime
import sys

from intelmq.lib import utils
from intelmq.lib.bot import Bot
from intelmq.lib.message import Event


def is_valid_date(strd):
    try:
        datetime.datetime.strptime(strd, '%Y%m%d')
        return True
    except Exception:
        return False


class MalwareDomainsParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report is None or not report.contains("raw"):
            self.acknowledge_message()
            return

        raw_report = utils.base64_decode(report.value("raw"))

        for row in raw_report.split('\n'):

            row = row.rstrip()

            if row.startswith("#") or len(row) == 0:
                continue

            values = row.split('\t')[1:]

            event = Event(report)

            event.add('source.fqdn', values[1], sanitize=True)
            event.add('event_description.text', values[2], sanitize=True)

            for i in range(4, len(values)):
                if is_valid_date(values[i]):
                    event.add('time.source',  # TODO: verify timezone
                              values[i] + "T00:00:00+00:00", sanitize=True, force=True)
                    break

{'fraction_non_alphanumeric': 0.11557788944723618, 'fraction_numerical': 0.013915732508697333, 'mean_word_length': 2.0554899645808735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2358234', 'n_tokens_mistral': 789, 'n_tokens_neox': 742, 'n_words': 179}