<commit_before>import StringIO, csv
from intelmq.lib.bot import Bot, sys
from intelmq.lib.event import Event

class PhishTankParserBot(Bot):

    def process(self):
        report = self.receive_message()
        report = report.strip()

        if report:
            columns = ["__IGNORE__", "url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"]
            rows = csv.DictReader(StringIO.StringIO(report), fieldnames = columns)

            for row in rows:
                event = Event()
                
                for key, value in row.items():

                    if key is "__IGNORE__":
                        continue

                    event.add(key, value)
                self.send_message(event)
                
        self.acknowledge_message()
<commit_msg>Update parser.py<commit_after>import StringIO, csv
from intelmq.lib.bot import Bot, sys
from intelmq.lib.event import Event

class PhishTankParserBot(Bot):

    def process(self):
        report = self.receive_message()
        report = report.strip()

        if report:
            columns = ["__IGNORE__", "url", "description_url", "source_time", "__IGNORE__", "__IGNORE__", "__IGNORE__", "target"] # ignore (__IGNORE__) fields specific to the source and other irrelevant fields
            rows = csv.DictReader(StringIO.StringIO(report), fieldnames = columns)

            for row in rows:
                event = Event()
                
                for key, value in row.items():

                    if key is "__IGNORE__":
                        continue

                    event.add(key, value)
                self.send_message(event)
                
        self.acknowledge_message()

{'fraction_non_alphanumeric': 0.09612044006948466, 'fraction_numerical': 0.0, 'mean_word_length': 1.8374384236453203, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4110338', 'n_tokens_mistral': 482, 'n_tokens_neox': 448, 'n_words': 122}