<commit_before>"""
Looks up malware hashes from virustotal.com and the Shadowserver hash
registry.

Maintainer: "<NAME>" <<EMAIL>>

Original code by <NAME>, 20110801
"""

import idiokit
import urllib2
import simplejson as json

from abusehelper.core import bot, events
from abusehelper.contrib.experts.combiner import Expert
import abusehelper.core.utils as utils


class MalwareHashExpert(Expert):
    virustotal_key = bot.Param(default="")
    hash_services = bot.Param(default=["virustotal", "shadowserver"])

    def __init__(self, *args, **keys):
        cache_time = keys.get('cache_time', 3600.0)
        Expert.__init__(self, *args, **keys)
        self.cache = utils.TimedCache(cache_time)

    def augment_keys(self, *args, **keys):
        yield (keys.get("resolve", ("md5",)))

    @idiokit.stream
    def query_virustotal(self, md5sum):
        try:
            url = "https://www.virustotal.com/vtapi/v2/file/report"
            parameters = {"resource": md5sum,
                          "apikey": self.virustotal_key}
            self.log.debug("Querying Virustotal for %r", md5sum)
            url = urllib2.Request(url, urllib.urlencode(parameters))
            info, fileobj = yield utils.fetch_url(url)
            data = fileobj.read()
            fileobj.close()

#            self.log.debug("Got data %r", data[:10])
            reports = json.loads(data)
#            self.log.debug("Parsed the report for %r", md5sum)
            if type(reports) != list:
                reports = [reports]

            if not reports:
                self.log.info("No data in report for %r", md5sum)
                return

            for report in reports:
                new = events.Event()
                if not report["response_code"]:
#                    self.log.debug("Not detected by virustotal: %r", md5sum)
                    new.add('av result', "none (virustotal.com)")
                else:
#                    self.log.debug("Detected by virustotal: %r", md5sum)
                    new.add("virustotal", report["permalink"])
                    new.add("sha1", report["sha1"])
                    new.add("last seen", "%s UTC" % report["scan_date"])

                    detections = report["positives"] * 100
                    total = report["total"]
                    new.add("av detection", str(detections / total))

                    for sc in report["scans"]:
                        detect, version, result, update = \
                            report["scans"][sc].values()
                        if not detect:
                            continue
                        new.add("av result",
                                "%s, %s %s (updated %s) (virustotal.com)" %
                                (result, sc, version, update))
#                self.log.debug("Done with Virustotal report for %r", md5sum)

                idiokit.stop(new)
        except utils.FetchUrlFailed, fuf:
            self.log.info("Virustotal query failed: %r",
                          ', '.join(fuf.args))
            raise
        except json.JSONDecodeError, e:
            self.log.info("Virustotal query failed: %r",
                          ', '.join(e.args))
            raise

    @idiokit.stream
    def query_shadowserver(self, md5sum):
        try:
            url = "http://innocuous.shadowserver.org/api/?query=" + \
                md5sum
            self.log.debug("Querying Shadowserver for %r", md5sum)
            info, fileobj = yield utils.fetch_url(url)
            data = fileobj.read()
            fileobj.close()

#            self.log.debug("Got data %r", repr(data))
            if data.startswith("! No match"):
                idiokit.stop(events.Event({'av result':
                                           "none (shadowserver.org)"}))
            elif data.startswith("! Whitelisted"):
                idiokit.stop(events.Event({'av result':
                                           "%s (shadowserver.com)" % data[2:]}))
            elif data.startswith("!"):
                idiokit.stop(events.Event({'av result':
                                           "API error with md5 %s" % (md5sum) +
                                           "(shadowserver.com)"}))

            details, vendordata = data.split("\n")[:2]
            vendors = json.loads(vendordata)
#            self.log.debug("Parsed the report for %s", md5sum)

            names  = ["md5", "sha1", "first seen", "last seen",
                      "file type", "ssdeep hash"]
            values = details.replace('"',"").split(",")[:6]

            new = events.Event()

            for vendor, name in vendors.iteritems():
                new.add("av result", "%s %s (shadowserver.org)" %
                        (name, vendor))

            for key, value in zip(names, values):
                if value:
                    if key.endswith('seen'):
                        value = "%s UTC" % (value)
                    new.add(key, value)
#            self.log.debug("Done with Shadowserver report for %r", md5sum)

            idiokit.stop(new)
        except utils.FetchUrlFailed, fuf:
            self.log.info("Shadowserver query failed: %r",
                          ', '.join(fuf.args))
            raise
        except json.JSONDecodeError, e:
            self.log.info("Shadowserver query failed: %r",
                          ', '.join(e.args))
            raise

    @idiokit.stream
    def augment(self, key):
        while True:
            eid, event = yield idiokit.next()

            for md5sum in event.values(key):
                # Only access md5 and sha1 sums
                if not (md5sum.isalnum() or
                        len(md5sum) == 32 or len(md5sum) == 40):
                    self.log.info("Not a md5 or sha1 hash: %r", md5sum)
                    continue

                if 'virustotal' in self.hash_services:
                    if not self.virustotal_key:
                        self.log.error("Virustotal API key not set!")
                        return
                    new = self.cache.get("virustotal" + md5sum, None)
                    if not new:
                        try:
                            new = yield self.query_virustotal(md5sum)
                            yield idiokit.send(eid, new)
                        except (utils.FetchUrlFailed, json.JSONDecodeError):
                            pass
                        self.cache.set("virustotal" + md5sum, new)

                if 'shadowserver' in self.hash_services:
                    new = self.cache.get("shadowserver" + md5sum, None)
                    if not new:
                        try:
                            new = yield self.query_shadowserver(md5sum)
                            yield idiokit.send(eid, new)
                        except (utils.FetchUrlFailed, json.JSONDecodeError):
                            pass
                        self.cache.set("shadowserver" + md5sum, new)


if __name__ == "__main__":
    MalwareHashExpert.from_command_line().execute()
<commit_msg>fixes unsatisfied import of urllib<commit_after>"""
Looks up malware hashes from virustotal.com and the Shadowserver hash
registry.

Maintainer: "<NAME>" <<EMAIL>>

Original code by <NAME>, 20110801
"""

import idiokit
import urllib2
import urllib
import simplejson as json

from abusehelper.core import bot, events
from abusehelper.contrib.experts.combiner import Expert
import abusehelper.core.utils as utils


class MalwareHashExpert(Expert):
    virustotal_key = bot.Param(default="")
    hash_services = bot.Param(default=["virustotal", "shadowserver"])

    def __init__(self, *args, **keys):
        cache_time = keys.get('cache_time', 3600.0)
        Expert.__init__(self, *args, **keys)
        self.cache = utils.TimedCache(cache_time)

    def augment_keys(self, *args, **keys):
        yield (keys.get("resolve", ("md5",)))

    @idiokit.stream
    def query_virustotal(self, md5sum):
        try:
            url = "https://www.virustotal.com/vtapi/v2/file/report"
            parameters = {"resource": md5sum,
                          "apikey": self.virustotal_key}
            self.log.debug("Querying Virustotal for %r", md5sum)
            url = urllib2.Request(url, urllib.urlencode(parameters))
            info, fileobj = yield utils.fetch_url(url)
            data = fileobj.read()
            fileobj.close()

#            self.log.debug("Got data %r", data[:10])
            reports = json.loads(data)
#            self.log.debug("Parsed the report for %r", md5sum)
            if type(reports) != list:
                reports = [reports]

            if not reports:
                self.log.info("No data in report for %r", md5sum)
                return

            for report in reports:
                new = events.Event()
                if not report["response_code"]:
#                    self.log.debug("Not detected by virustotal: %r", md5sum)
                    new.add('av result', "none (virustotal.com)")
                else:
#                    self.log.debug("Detected by virustotal: %r", md5sum)
                    new.add("virustotal", report["permalink"])
                    new.add("sha1", report["sha1"])
                    new.add("last seen", "%s UTC" % report["scan_date"])

                    detections = report["positives"] * 100
                    total = report["total"]
                    new.add("av detection", str(detections / total))

                    for sc in report["scans"]:
                        detect, version, result, update = \
                            report["scans"][sc].values()
                        if not detect:
                            continue
                        new.add("av result",
                                "%s, %s %s (updated %s) (virustotal.com)" %
                                (result, sc, version, update))
#                self.log.debug("Done with Virustotal report for %r", md5sum)

                idiokit.stop(new)
        except utils.FetchUrlFailed, fuf:
            self.log.info("Virustotal query failed: %r",
                          ', '.join(fuf.args))
            raise
        except json.JSONDecodeError, e:
            self.log.info("Virustotal query failed: %r",
                          ', '.join(e.args))
            raise

    @idiokit.stream
    def query_shadowserver(self, md5sum):
        try:
            url = "http://innocuous.shadowserver.org/api/?query=" + \
                md5sum
            self.log.debug("Querying Shadowserver for %r", md5sum)
            info, fileobj = yield utils.fetch_url(url)
            data = fileobj.read()
            fileobj.close()

#            self.log.debug("Got data %r", repr(data))
            if data.startswith("! No match"):
                idiokit.stop(events.Event({'av result':
                                           "none (shadowserver.org)"}))
            elif data.startswith("! Whitelisted"):
                idiokit.stop(events.Event({'av result':
                                           "%s (shadowserver.com)" % data[2:]}))
            elif data.startswith("!"):
                idiokit.stop(events.Event({'av result':
                                           "API error with md5 %s" % (md5sum) +
                                           "(shadowserver.com)"}))

            details, vendordata = data.split("\n")[:2]
            vendors = json.loads(vendordata)
#            self.log.debug("Parsed the report for %s", md5sum)

            names  = ["md5", "sha1", "first seen", "last seen",
                      "file type", "ssdeep hash"]
            values = details.replace('"',"").split(",")[:6]

            new = events.Event()

            for vendor, name in vendors.iteritems():
                new.add("av result", "%s %s (shadowserver.org)" %
                        (name, vendor))

            for key, value in zip(names, values):
                if value:
                    if key.endswith('seen'):
                        value = "%s UTC" % (value)
                    new.add(key, value)
#            self.log.debug("Done with Shadowserver report for %r", md5sum)

            idiokit.stop(new)
        except utils.FetchUrlFailed, fuf:
            self.log.info("Shadowserver query failed: %r",
                          ', '.join(fuf.args))
            raise
        except json.JSONDecodeError, e:
            self.log.info("Shadowserver query failed: %r",
                          ', '.join(e.args))
            raise

    @idiokit.stream
    def augment(self, key):
        while True:
            eid, event = yield idiokit.next()

            for md5sum in event.values(key):
                # Only access md5 and sha1 sums
                if not (md5sum.isalnum() or
                        len(md5sum) == 32 or len(md5sum) == 40):
                    self.log.info("Not a md5 or sha1 hash: %r", md5sum)
                    continue

                if 'virustotal' in self.hash_services:
                    if not self.virustotal_key:
                        self.log.error("Virustotal API key not set!")
                        return
                    new = self.cache.get("virustotal" + md5sum, None)
                    if not new:
                        try:
                            new = yield self.query_virustotal(md5sum)
                            yield idiokit.send(eid, new)
                        except (utils.FetchUrlFailed, json.JSONDecodeError):
                            pass
                        self.cache.set("virustotal" + md5sum, new)

                if 'shadowserver' in self.hash_services:
                    new = self.cache.get("shadowserver" + md5sum, None)
                    if not new:
                        try:
                            new = yield self.query_shadowserver(md5sum)
                            yield idiokit.send(eid, new)
                        except (utils.FetchUrlFailed, json.JSONDecodeError):
                            pass
                        self.cache.set("shadowserver" + md5sum, new)


if __name__ == "__main__":
    MalwareHashExpert.from_command_line().execute()

{'fraction_non_alphanumeric': 0.11591951994352277, 'fraction_numerical': 0.00889516413695729, 'mean_word_length': 1.4095934682769178, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2481921', 'n_tokens_mistral': 3987, 'n_tokens_neox': 3797, 'n_words': 902}