<commit_before><commit_msg>Added examples/standalone/custom/ircfeed.sanitizer.py, and example sanitizer for ircfeedbot.<commit_after>import re
from abusehelper.core import events, config

sanitizer = config.load_module("sanitizer")
dnsbl = re.compile("(?:^|,)([^,]+):[^0]").findall

class IRCFeedSanitizer(sanitizer.Sanitizer):
    def sanitize(self, event):
        new = events.Event()
        new.update("ip", event.values("ip", sanitizer.ip))
        new.update("asn", event.values("asn"))
        new.add("time", sanitizer.format_time())
        
        for types in event.values("dnsbl", dnsbl):
            new.update("type", types)
        if not new.contains("type"):
            new.add("type", "unknown")
            
        if not new.contains("ip"):
            self.log.error("No valid IP for event %r", event)
            return []
        return [new]

if __name__ == "__main__":
    IRCFeedSanitizer.from_command_line().execute()
{'fraction_non_alphanumeric': 0.1550632911392405, 'fraction_numerical': 0.0010548523206751054, 'mean_word_length': 2.9873949579831933, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3099378', 'n_tokens_mistral': 303, 'n_tokens_neox': 299, 'n_words': 60}