<commit_before>import re

import watchme

class Watchme_filter_default_syslog(object):
    def __init__(self):
        pass

    def filter(self, evt):
        # Severity
        # system
        if "Kernel panic" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "Hardware Error" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "[sudo]" in evt.content:
            if "session opened" in evt.content:
                evt.severity = watchme.SEVERITY_MEDIUM
            elif ("incorrect password" in evt.content
                    or "pam_unix(sudo:auth)" in evt.content):
                evt.severity = watchme.SEVERITY_HIGH
            else:
                evt.severity = watchme.SEVERITY_LOW
        if "segfault" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        # devices
        if "New USB device found" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if "USB disconnect" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        # network
        if "Malformed Packet" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "adding default route" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if "authenticated" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if any(x in evt.content for x in ["authenticated", "associated", "carrier acquired", "deauthenticating from", "carrier lost"]):
            evt.severity = watchme.SEVERITY_MEDIUM
        return evt

class Watchme_filter_default_syslog_iptables(object):
    def __init__(self):
        pass

    def filter(self, evt):
        if "iptables-drop:" in evt.content:
            action="drop"
        elif "iptables-accept:" in evt.content:
            action="accept"
        else:
            return evt # not for us

        # Content rewording
        m_ip = re.match(r".* iptables-(?P<action>[a-z]*): IN=(?P<iface_in>\w*) OUT=(?P<iface_out>\w*) MAC=(?P<mac>[0-9a-f:]*) SRC=(?P<ip_src>[0-9\.]*) DST=(?P<ip_dst>[0-9\.]*) LEN=(?P<len>[0-9]*) TOS=(?P<tos>[0-9a-zx]*) PREC=(?P<prec>[0-9a-zx]*) TTL=(?P<ttl>[0-9]*) ID=(?P<id>[0-9]*) .* PROTO=(?P<proto>[0-9A-Z]*)", evt.content)
        if m_ip:
            d_ip = m_ip.groupdict()
            mac_src = d_ip['mac'][18:35]
            mac_dst = d_ip['mac'][:17]
            d_ip['mac'] = "%s > %s" % (mac_src, mac_dst)
            if d_ip['proto'] == "TCP":
                m_tcp = re.match(r".* SPT=(?P<port_src>[0-9]*) DPT=(?P<port_dst>[0-9]*) WINDOW=(?P<win>[0-9]*) RES=(?P<res>[0-9a-zx]*) (?P<flags>.*) URGP=(?P<urgp>[0-9]*)", evt.content)
                if m_tcp:
                    d = m_tcp.groupdict()
                    evt.content = "%s: %s %s %s:%s > %s:%s %s [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d['port_src'], d_ip['ip_dst'], d['port_dst'], d['flags'], d_ip['len'], d_ip['ttl'])
            elif d_ip['proto'] == "UDP":
                m_udp = re.match(r".* SPT=(?P<port_src>[0-9]*) DPT=(?P<port_dst>[0-9]*) LEN=(?P<len>[0-9]*)", evt.content)
                if m_udp:
                    d = m_udp.groupdict()
                    evt.content = "%s: %s %s %s:%s > %s:%s UDP [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d['port_src'], d_ip['ip_dst'], d['port_dst'], d_ip['len'], d_ip['ttl'])
            elif d_ip['proto'] == "ICMP":
                m_icmp = re.match(r".* TYPE=(?P<type>[0-9]*) CODE=(?P<code>[0-9]*) ID=(?P<id>[0-9]*) SEQ=(?P<seq>[0-9]*)", evt.content)
                if m_icmp:
                    d = m_icmp.groupdict()
                    evt.content = "%s: %s %s %s > %s ICMP %s-%s [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d_ip['ip_dst'], d['type'], d['code'], d_ip['len'], d_ip['ttl'])
            else:
                evt.content = "%s: %s %s %s > %s %s [%s] %s" % (
                    action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d_ip['ip_dst'], d_ip['proto'], d_ip['len'], d_ip['ttl'])

        # Severity
        if action == "drop":
            if m_ip:
                if mac_dst == "ff:ff:ff:ff:ff:ff": evt.severity = watchme.SEVERITY_LOW
                elif d_ip['ip_dst'].endswith("255"): evt.severity = watchme.SEVERITY_LOW
                elif d_ip['ip_dst'].startswith("224.0.0"): evt.severity = watchme.SEVERITY_LOW
            if evt.severity == watchme.SEVERITY_UNINITIALIZED:
                evt.severity = watchme.SEVERITY_HIGH
        elif action == "accept":
            evt.severity = watchme.SEVERITY_INFO

        return evt

class Watchme_filter_default_arpwatch(object):
    def __init__(self):
        pass

    def filter(self, evt):
        # Severity
        if not "[arpwatch]" in evt.content:
            return evt # not for us

        if any(x in evt.content for x in ["Wrote pid", "Running as", "listening on", "new station"]):
            evt.severity = watchme.SEVERITY_LOW
        else:
            evt.severity = watchme.SEVERITY_HIGH
        return evt
<commit_msg>more filters<commit_after>import re

import watchme

class Watchme_filter_default_syslog(object):
    def __init__(self):
        pass

    def filter(self, evt):
        # Severity
        # system
        if "Kernel panic" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "Hardware Error" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "[sudo]" in evt.content:
            if "session opened" in evt.content:
                evt.severity = watchme.SEVERITY_MEDIUM
            elif ("incorrect password" in evt.content
                    or "pam_unix(sudo:auth)" in evt.content):
                evt.severity = watchme.SEVERITY_HIGH
            else:
                evt.severity = watchme.SEVERITY_LOW
        if "segfault" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "Out of memory" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "Killed process" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        # devices
        if "New USB device found" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if "USB disconnect" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        # network
        if "Malformed Packet" in evt.content:
            evt.severity = watchme.SEVERITY_HIGH
        if "adding default route" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if "authenticated" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        if any(x in evt.content for x in ["authenticated", "associated", "carrier acquired", "deauthenticating from", "carrier lost"]):
            evt.severity = watchme.SEVERITY_MEDIUM
        return evt

class Watchme_filter_default_syslog_iptables(object):
    def __init__(self):
        pass

    def filter(self, evt):
        if "iptables-drop:" in evt.content:
            action="drop"
        elif "iptables-accept:" in evt.content:
            action="accept"
        else:
            return evt # not for us

        # Content rewording
        m_ip = re.match(r".* iptables-(?P<action>[a-z]*): IN=(?P<iface_in>\w*) OUT=(?P<iface_out>\w*) MAC=(?P<mac>[0-9a-f:]*) SRC=(?P<ip_src>[0-9\.]*) DST=(?P<ip_dst>[0-9\.]*) LEN=(?P<len>[0-9]*) TOS=(?P<tos>[0-9a-zx]*) PREC=(?P<prec>[0-9a-zx]*) TTL=(?P<ttl>[0-9]*) ID=(?P<id>[0-9]*) .* PROTO=(?P<proto>[0-9A-Z]*)", evt.content)
        if m_ip:
            d_ip = m_ip.groupdict()
            mac_src = d_ip['mac'][18:35]
            mac_dst = d_ip['mac'][:17]
            d_ip['mac'] = "%s > %s" % (mac_src, mac_dst)
            if d_ip['proto'] == "TCP":
                m_tcp = re.match(r".* SPT=(?P<port_src>[0-9]*) DPT=(?P<port_dst>[0-9]*) WINDOW=(?P<win>[0-9]*) RES=(?P<res>[0-9a-zx]*) (?P<flags>.*) URGP=(?P<urgp>[0-9]*)", evt.content)
                if m_tcp:
                    d = m_tcp.groupdict()
                    evt.content = "%s: %s %s %s:%s > %s:%s %s [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d['port_src'], d_ip['ip_dst'], d['port_dst'], d['flags'], d_ip['len'], d_ip['ttl'])
            elif d_ip['proto'] == "UDP":
                m_udp = re.match(r".* SPT=(?P<port_src>[0-9]*) DPT=(?P<port_dst>[0-9]*) LEN=(?P<len>[0-9]*)", evt.content)
                if m_udp:
                    d = m_udp.groupdict()
                    evt.content = "%s: %s %s %s:%s > %s:%s UDP [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d['port_src'], d_ip['ip_dst'], d['port_dst'], d_ip['len'], d_ip['ttl'])
            elif d_ip['proto'] == "ICMP":
                m_icmp = re.match(r".* TYPE=(?P<type>[0-9]*) CODE=(?P<code>[0-9]*) ID=(?P<id>[0-9]*) SEQ=(?P<seq>[0-9]*)", evt.content)
                if m_icmp:
                    d = m_icmp.groupdict()
                    evt.content = "%s: %s %s %s > %s ICMP %s-%s [%s] %s" % (
                        action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d_ip['ip_dst'], d['type'], d['code'], d_ip['len'], d_ip['ttl'])
            else:
                evt.content = "%s: %s %s %s > %s %s [%s] %s" % (
                    action, d_ip['iface_in'], d_ip['mac'], d_ip['ip_src'], d_ip['ip_dst'], d_ip['proto'], d_ip['len'], d_ip['ttl'])

        # Severity
        if action == "drop":
            if m_ip:
                if mac_dst == "ff:ff:ff:ff:ff:ff": evt.severity = watchme.SEVERITY_LOW
                elif d_ip['ip_dst'].endswith("255"): evt.severity = watchme.SEVERITY_LOW
                elif d_ip['ip_dst'].startswith("224.0.0"): evt.severity = watchme.SEVERITY_LOW
            if evt.severity == watchme.SEVERITY_UNINITIALIZED:
                evt.severity = watchme.SEVERITY_HIGH
        elif action == "accept":
            evt.severity = watchme.SEVERITY_INFO

        return evt

class Watchme_filter_default_arpwatch(object):
    def __init__(self):
        pass

    def filter(self, evt):
        # Severity
        if not "[arpwatch]" in evt.content:
            return evt # not for us
        if any(x in evt.content for x in ["Wrote pid", "Running as", "listening on"]):
            return evt # not interesting
        if "new station" in evt.content:
            evt.severity = watchme.SEVERITY_MEDIUM
        else:
            evt.severity = watchme.SEVERITY_HIGH

        return evt

{'fraction_non_alphanumeric': 0.17006673021925645, 'fraction_numerical': 0.010676835081029552, 'mean_word_length': 2.094690265486726, 'pattern_counts': {'":': 12, '<': 53, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3698066', 'n_tokens_mistral': 3908, 'n_tokens_neox': 3868, 'n_words': 743}