<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_malcheck
# Purpose:      Checks if an ASN, IP or domain is malicious.
#
# Author:       <EMAIL>
#
# Created:     14/12/2013
# Copyright:   (c) <NAME>, 2013
# Licence:     GPL
# -------------------------------------------------------------------------------

from netaddr import IPAddress, IPNetwork
import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

malchecks = {
    'abuse.ch Zeus Tracker (Domain)': {
        'id': 'abusezeusdomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://zeustracker.abuse.ch/blocklist.php?download=baddomains'
    },
    'abuse.ch Zeus Tracker (IP)': {
        'id': 'abusezeusip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://zeustracker.abuse.ch/blocklist.php?download=badips'
    },
    'abuse.ch Feodo Tracker (Domain)': {
        'id': 'abusefeododomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist'
    },
    'abuse.ch Feodo Tracker (IP)': {
        'id': 'abusefeodoip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist'
    },
    'abuse.ch Palevo Tracker (Domain)': {
        'id': 'abusepalevodomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist'
    },
    'abuse.ch Palevo Tracker (IP)': {
        'id': 'abusepalevoip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://palevotracker.abuse.ch/blocklists.php?download=ipblocklist'
    },
    'abuse.ch SSL Blacklist (IP)': {
        'id': 'abusesslblip',
        'type': 'list',
        'checks': ['ip'],
        'url': 'https://sslbl.abuse.ch/blacklist/sslipblacklist.csv',
        'regex': '{0},.*'
    },
    'Google SafeBrowsing (Domain/IP)': {
        'id': 'googledomain',
        'type': 'query',
        'checks': ['domain', 'ip'],
        'url': 'http://www.google.com/safebrowsing/diagnostic?site={0}',
        'badregex': ['.*may harm your computer.*',
                     '.*this site has hosted malicious software.*'
                     ],
        'goodregex': []
    },
    'Google SafeBrowsing (ASN)': {
        'id': 'googleasn',
        'type': 'query',
        'checks': ['asn'],
        'url': 'http://www.google.com/safebrowsing/diagnostic?site=AS:{0}',
        'badregex': ['.*for example.*, that appeared to function as intermediaries.*',
                     '.*this network has hosted sites that have distributed malicious.*'
                     ],
        'goodregex': []
    },
    'McAfee Site Advisor': {
        'id': 'mcafeedomain',
        'type': 'query',
        'checks': ['domain'],
        'url': 'http://www.siteadvisor.com/sites/{0}',
        'badregex': ['.*This link might be dangerous.*'],
        'goodregex': []
    },
    'AVG Safety Report': {
        'id': 'avgdomain',
        'type': 'query',
        'checks': ['domain'],
        'url': 'http://www.avgthreatlabs.com/website-safety-reports/domain/{0}',
        'badregex': ['.*potentially active malware was detected.*'],
        'goodregex': []
    },
    'malwaredomainlist.com IP List': {
        'id': 'malwaredomainlistip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.malwaredomainlist.com/hostslist/ip.txt'
    },
    'malwaredomainlist.com Domain List': {
        'id': 'malwaredomainlistdomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',
        'regex': '.*\s+{0}[\s$]'
    },
    'PhishTank': {
        'id': 'phishtank',
        'type': 'list',
        'checks': ['domain'],
        'url': 'http://data.phishtank.com/data/online-valid.csv',
        'regex': '\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'
    },
    'malc0de.com List': {
        'id': 'malc0de',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://malc0de.com/bl/IP_Blacklist.txt'
    },
    'TOR Node List': {
        'id': 'tornodes',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://torstatus.blutmagie.de/ip_list_all.php/Tor_ip_list_ALL.csv'
    },
    'blocklist.de List': {
        'id': 'blocklistde',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://lists.blocklist.de/lists/all.txt'
    },
    'Autoshun.org List': {
        'id': 'autoshun',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.autoshun.org/files/shunlist.csv',
        'regex': '{0},.*'
    },
    'Internet Storm Center': {
        'id': 'isc',
        'type': 'query',
        'checks': ['ip'],
        'url': 'https://isc.sans.edu/api/ip/{0}',
        'badregex': ['.*attacks.*'],
        'goodregex': []
    },
    'AlienVault IP Reputation Database': {
        'id': 'alienvault',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://reputation.alienvault.com/reputation.generic',
        'regex': '{0} #.*'
    },
    'OpenBL.org Blacklist': {
        'id': 'openbl',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.openbl.org/lists/base.txt'
    },
    'ThreatExpert.com Database': {
        'id': 'threatexpert',
        'type': 'query',
        'checks': ['ip', 'domain'],
        'url': 'http://www.threatexpert.com/reports.aspx?find={0}&tf=3',
        'badregex': ['.*<strong>Findings</strong>.*'],
        'goodregex': []
    },
    'TotalHash.com Database': {<commit_msg>Fixed bug when handling ISC output.<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_malcheck
# Purpose:      Checks if an ASN, IP or domain is malicious.
#
# Author:       <EMAIL>
#
# Created:     14/12/2013
# Copyright:   (c) <NAME>, 2013
# Licence:     GPL
# -------------------------------------------------------------------------------

from netaddr import IPAddress, IPNetwork
import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

malchecks = {
    'abuse.ch Zeus Tracker (Domain)': {
        'id': 'abusezeusdomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://zeustracker.abuse.ch/blocklist.php?download=baddomains'
    },
    'abuse.ch Zeus Tracker (IP)': {
        'id': 'abusezeusip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://zeustracker.abuse.ch/blocklist.php?download=badips'
    },
    'abuse.ch Feodo Tracker (Domain)': {
        'id': 'abusefeododomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist'
    },
    'abuse.ch Feodo Tracker (IP)': {
        'id': 'abusefeodoip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://feodotracker.abuse.ch/blocklist/?download=ipblocklist'
    },
    'abuse.ch Palevo Tracker (Domain)': {
        'id': 'abusepalevodomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist'
    },
    'abuse.ch Palevo Tracker (IP)': {
        'id': 'abusepalevoip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://palevotracker.abuse.ch/blocklists.php?download=ipblocklist'
    },
    'abuse.ch SSL Blacklist (IP)': {
        'id': 'abusesslblip',
        'type': 'list',
        'checks': ['ip'],
        'url': 'https://sslbl.abuse.ch/blacklist/sslipblacklist.csv',
        'regex': '{0},.*'
    },
    'Google SafeBrowsing (Domain/IP)': {
        'id': 'googledomain',
        'type': 'query',
        'checks': ['domain', 'ip'],
        'url': 'http://www.google.com/safebrowsing/diagnostic?site={0}',
        'badregex': ['.*may harm your computer.*',
                     '.*this site has hosted malicious software.*'
                     ],
        'goodregex': []
    },
    'Google SafeBrowsing (ASN)': {
        'id': 'googleasn',
        'type': 'query',
        'checks': ['asn'],
        'url': 'http://www.google.com/safebrowsing/diagnostic?site=AS:{0}',
        'badregex': ['.*for example.*, that appeared to function as intermediaries.*',
                     '.*this network has hosted sites that have distributed malicious.*'
                     ],
        'goodregex': []
    },
    'McAfee Site Advisor': {
        'id': 'mcafeedomain',
        'type': 'query',
        'checks': ['domain'],
        'url': 'http://www.siteadvisor.com/sites/{0}',
        'badregex': ['.*This link might be dangerous.*'],
        'goodregex': []
    },
    'AVG Safety Report': {
        'id': 'avgdomain',
        'type': 'query',
        'checks': ['domain'],
        'url': 'http://www.avgthreatlabs.com/website-safety-reports/domain/{0}',
        'badregex': ['.*potentially active malware was detected.*'],
        'goodregex': []
    },
    'malwaredomainlist.com IP List': {
        'id': 'malwaredomainlistip',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.malwaredomainlist.com/hostslist/ip.txt'
    },
    'malwaredomainlist.com Domain List': {
        'id': 'malwaredomainlistdomain',
        'type': 'list',
        'checks': ['domain'],
        'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',
        'regex': '.*\s+{0}[\s$]'
    },
    'PhishTank': {
        'id': 'phishtank',
        'type': 'list',
        'checks': ['domain'],
        'url': 'http://data.phishtank.com/data/online-valid.csv',
        'regex': '\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'
    },
    'malc0de.com List': {
        'id': 'malc0de',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://malc0de.com/bl/IP_Blacklist.txt'
    },
    'TOR Node List': {
        'id': 'tornodes',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://torstatus.blutmagie.de/ip_list_all.php/Tor_ip_list_ALL.csv'
    },
    'blocklist.de List': {
        'id': 'blocklistde',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://lists.blocklist.de/lists/all.txt'
    },
    'Autoshun.org List': {
        'id': 'autoshun',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.autoshun.org/files/shunlist.csv',
        'regex': '{0},.*'
    },
    'Internet Storm Center': {
        'id': 'isc',
        'type': 'query',
        'checks': ['ip'],
        'url': 'https://isc.sans.edu/api/ip/{0}',
        'badregex': ['.*<attacks>.*'],
        'goodregex': []
    },
    'AlienVault IP Reputation Database': {
        'id': 'alienvault',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'https://reputation.alienvault.com/reputation.generic',
        'regex': '{0} #.*'
    },
    'OpenBL.org Blacklist': {
        'id': 'openbl',
        'type': 'list',
        'checks': ['ip', 'netblock'],
        'url': 'http://www.openbl.org/lists/base.txt'
    },
    'ThreatExpert.com Database': {
        'id': 'threatexpert',
        'type': 'query',
        'checks': ['ip', 'domain'],
        'url': 'http://www.threatexpert.com/reports.aspx?find={0}&tf=3',
        'badregex': ['.*<strong>Findings</strong>.*'],
        'goodregex': []
    },
    'TotalHash.com Database': {
{'fraction_non_alphanumeric': 0.2354596622889306, 'fraction_numerical': 0.005116834385127068, 'mean_word_length': 2.0325833979829326, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 18, 'lorem ipsum': 0, 'www.': 20, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1842418', 'n_tokens_mistral': 4103, 'n_tokens_neox': 3956, 'n_words': 719}