<commit_before># -*- coding: utf-8 -*-
"""
Copyright (C) 2016 by Bundesamt für Sicherheit in der Informationstechnik

Software engineering by Intevation GmbH

This is a configuration File for the shadowserver parser

Mappings are "straight forward" each mapping is a dict
of at least three keys:

1. required fields:
   the parser will work this keys first.
2. optional fields:
   the parser will try to interpret these values.
   if it fails, the value is written to the extra field
3. constant fields:
   Some information about an event may not be explicitly stated in a
   feed because it is implicit in the nature of the feed. For instance
   a feed that is exclusively about HTTP may not have a field for the
   protocol because it's always TCP.

The first value is the IntelMQ key,
the second value is the row in the shadowserver csv.


Reference material:
    * when setting the classification.* fields, please use the taxonomy from
      [eCSIRT II](https://www.trusted-introducer.org/Incident-Classification-Taxonomy.pdf)
      Also to be found on the
      [ENISA page](https://www.enisa.europa.eu/topics/csirt-cert-services/community-projects/existing-taxonomies)
    * please respect the Data harmonization ontology: https://github.com/certtools/intelmq/blob/master/docs/Data-Harmonization.md


TODOs:
    There is a bunch of inline todos.
    Most of them show lines of code were the mapping  has to be validated

    @ Check-Implementation Tags for parser configs.
    dmth thinks it's not sufficient. Some CERT-Expertise is needed to
    check if the mappings are correct.

"""
import intelmq.lib.harmonization as harmonization


def get_feed(feedname):
    # TODO should this be case insensitive?
    feed_idx = {
        "Accessible-Cisco-Smart-Install": accessible_cisco_smart_install,
        "Accessible-CWMP": accessible_cwmp,
        "Accessible-RDP": accessible_rdp,
        "Accessible-SMB": accessible_smb,
        "Accessible-Telnet": accessible_telnet,
        "Accessible-VNC": accessible_vnc,
        "Blacklisted-IP": blacklisted_ip,
        "Botnet-Drone-Hadoop": botnet_drone_hadoop,
        "Compromised-Website": compromised_website,
        "DNS-open-resolvers": dns_open_resolvers,
        "Microsoft-Sinkhole": microsoft_sinkhole,
        "NTP-Monitor": ntp_monitor,
        "NTP-Version": ntp_version,
        "Open-Chargen": open_chargen,
        "Open-Elasticsearch": open_elasticsearch,
        "Open-IPMI": open_ipmi,
        "Open-LDAP": open_ldap,
        "Open-mDNS": open_mdns,
        "Open-Memcached": open_memcached,
        "Open-MongoDB": open_mongodb,
        "Open-MSSQL": open_mssql,
        "Open-NATPMP": open_natpmp,
        "Open-NetBIOS": open_netbios,
        "Open-Netis": open_netis,
        "Open-Portmapper": open_portmapper,
        "Open-QOTD": open_qotd,
        "Open-Redis": open_redis,
        "Open-SNMP": open_snmp,
        "Open-SSDP": open_ssdp,
        "Open-TFTP": open_tftp,
        "Open-XDMCP": open_xdmcp,
        "Sandbox-URL": sandbox_url,
        "Sinkhole-HTTP-Drone": sinkhole_http_drone,
        "Spam-URL": spam_url,
        "Ssl-Freak-Scan": ssl_freak_scan,  # Only differs in a few extra fields
        "Ssl-Scan": ssl_scan,  # a.k.a POODLE
        "Vulnerable-ISAKMP": vulnerable_isakmp,
    }

    return feed_idx.get(feedname)


def add_UTC_to_timestamp(value):
    return value + ' UTC'


def convert_bool(value):
    if value.lower() in ('y', 'yes', 'true', 'enabled'):
        return True
    elif value.lower() in ('n', 'no', 'false', 'disabled'):
        return False


def validate_to_none(value):
    if not len(value) or value in ['0', 'unknown']:
        return None
    return value


def convert_int(value):
    """ Returns an int or None for empty strings. """
    if not value:
        return None
    else:
        return int(value)


def convert_float(value):
    """ Returns an float or None for empty strings. """
    if not value:
        return None
    else:
        return float(value)


def convert_hostname_and_url(value, row):
    """
    URLs are split into hostname and path, we can also guess the protocol here.
    but only guess if the protocol is in a set of known good values.
    """
    if row['application'] in ['http', 'https', 'irc']:
        if row['hostname'] and row['url']:
            url = row['url'] if row['url'].startswith('/') else '/' + row['url']
            return row['application'] + '://' + row['hostname'] + url

        elif row['hostname'] and not row['url']:
            return row['application'] + '://' + row['hostname']

    return value


def convert_httphost_and_url(value, row):
    """
    URLs are split into hostname and path, we can also guess the protocol here.
    """
    if row['http_host'] and row['url']:
        return 'http://' + row['http_host'] + row['url']
    return value


def invalidate_zero(value):
    """ Returns an int or None for empty strings or '0'. """
    if not value:
        return None
    elif int(value) != 0:
        return int(value)


# TODO this function is a wild guess...
def set_tor_node(value):<commit_msg>Fix convert_httphost_and_url<commit_after># -*- coding: utf-8 -*-
"""
Copyright (C) 2016 by Bundesamt für Sicherheit in der Informationstechnik

Software engineering by Intevation GmbH

This is a configuration File for the shadowserver parser

Mappings are "straight forward" each mapping is a dict
of at least three keys:

1. required fields:
   the parser will work this keys first.
2. optional fields:
   the parser will try to interpret these values.
   if it fails, the value is written to the extra field
3. constant fields:
   Some information about an event may not be explicitly stated in a
   feed because it is implicit in the nature of the feed. For instance
   a feed that is exclusively about HTTP may not have a field for the
   protocol because it's always TCP.

The first value is the IntelMQ key,
the second value is the row in the shadowserver csv.


Reference material:
    * when setting the classification.* fields, please use the taxonomy from
      [eCSIRT II](https://www.trusted-introducer.org/Incident-Classification-Taxonomy.pdf)
      Also to be found on the
      [ENISA page](https://www.enisa.europa.eu/topics/csirt-cert-services/community-projects/existing-taxonomies)
    * please respect the Data harmonization ontology: https://github.com/certtools/intelmq/blob/master/docs/Data-Harmonization.md


TODOs:
    There is a bunch of inline todos.
    Most of them show lines of code were the mapping  has to be validated

    @ Check-Implementation Tags for parser configs.
    dmth thinks it's not sufficient. Some CERT-Expertise is needed to
    check if the mappings are correct.

"""
import intelmq.lib.harmonization as harmonization
import re

def get_feed(feedname):
    # TODO should this be case insensitive?
    feed_idx = {
        "Accessible-Cisco-Smart-Install": accessible_cisco_smart_install,
        "Accessible-CWMP": accessible_cwmp,
        "Accessible-RDP": accessible_rdp,
        "Accessible-SMB": accessible_smb,
        "Accessible-Telnet": accessible_telnet,
        "Accessible-VNC": accessible_vnc,
        "Blacklisted-IP": blacklisted_ip,
        "Botnet-Drone-Hadoop": botnet_drone_hadoop,
        "Compromised-Website": compromised_website,
        "DNS-open-resolvers": dns_open_resolvers,
        "Microsoft-Sinkhole": microsoft_sinkhole,
        "NTP-Monitor": ntp_monitor,
        "NTP-Version": ntp_version,
        "Open-Chargen": open_chargen,
        "Open-Elasticsearch": open_elasticsearch,
        "Open-IPMI": open_ipmi,
        "Open-LDAP": open_ldap,
        "Open-mDNS": open_mdns,
        "Open-Memcached": open_memcached,
        "Open-MongoDB": open_mongodb,
        "Open-MSSQL": open_mssql,
        "Open-NATPMP": open_natpmp,
        "Open-NetBIOS": open_netbios,
        "Open-Netis": open_netis,
        "Open-Portmapper": open_portmapper,
        "Open-QOTD": open_qotd,
        "Open-Redis": open_redis,
        "Open-SNMP": open_snmp,
        "Open-SSDP": open_ssdp,
        "Open-TFTP": open_tftp,
        "Open-XDMCP": open_xdmcp,
        "Sandbox-URL": sandbox_url,
        "Sinkhole-HTTP-Drone": sinkhole_http_drone,
        "Spam-URL": spam_url,
        "Ssl-Freak-Scan": ssl_freak_scan,  # Only differs in a few extra fields
        "Ssl-Scan": ssl_scan,  # a.k.a POODLE
        "Vulnerable-ISAKMP": vulnerable_isakmp,
    }

    return feed_idx.get(feedname)


def add_UTC_to_timestamp(value):
    return value + ' UTC'


def convert_bool(value):
    if value.lower() in ('y', 'yes', 'true', 'enabled'):
        return True
    elif value.lower() in ('n', 'no', 'false', 'disabled'):
        return False


def validate_to_none(value):
    if not len(value) or value in ['0', 'unknown']:
        return None
    return value


def convert_int(value):
    """ Returns an int or None for empty strings. """
    if not value:
        return None
    else:
        return int(value)


def convert_float(value):
    """ Returns an float or None for empty strings. """
    if not value:
        return None
    else:
        return float(value)


def convert_hostname_and_url(value, row):
    """
    URLs are split into hostname and path, we can also guess the protocol here.
    but only guess if the protocol is in a set of known good values.
    """
    if row['application'] in ['http', 'https', 'irc']:
        if row['hostname'] and row['url']:
            url = row['url'] if row['url'].startswith('/') else '/' + row['url']
            return row['application'] + '://' + row['hostname'] + url

        elif row['hostname'] and not row['url']:
            return row['application'] + '://' + row['hostname']

    return value


def convert_httphost_and_url(value, row):
    """
    URLs are split into hostname and path, we can also guess the protocol here.
    With some reports, url/http_url holds only the path, with others the full HTTP request.
    """
    if "url" in row:
        if row['http_host'] and row['url']:
            path = re.sub(r'^[^/]*', '', row['url'])
            path = re.sub(r'\s.*$', '', path)
            return 'http://' + row['http_host'] + path
    elif "http_url" in row:
        if row['http_host'] and row['http_url']:
            path = re.sub(r'^[^/]*', '', row['http_url'])
            path = re.sub(r'\s.*$', '', path)
            return 'http://' + row['http_host'] + path
    return value


def invalidate_zero(value):
    """ Returns an int or None for empty strings or '0'. """
    if not value:
        return None
    elif int(value) != 0:
        return int(value)


# TODO this function is a wild guess...
def set_tor_node(value):
{'fraction_non_alphanumeric': 0.11219329462446151, 'fraction_numerical': 0.002060310919647874, 'mean_word_length': 2.851063829787234, 'pattern_counts': {'":': 74, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5553012', 'n_tokens_mistral': 3411, 'n_tokens_neox': 3309, 'n_words': 1046}