import clamd
import requests

clamd_socket = clamd.ClamdUnixSocket()


def scan_html(url):
    """
    todo: should crawl the page, using lxml then scan each page's content
    """
    return clamd_socket.scan_stream(requests.get(url, timeout=10.0).content)

if __name__ == "__main__":
    print scan_html("https://secure.eicar.org/eicar.com")

import clamd
import requests

clamd_socket = clamd.ClamdUnixSocket()


def crawl_html(url):
    """
    Naive html crawler.

    To be truly effective we need to determine what files might actually effect
    the rendering of the page and what files a user might be tricked into
    downloading.

    * JavaScript in <script> tags
    * Executable files

    Things we might not be interested in:
    * images
    * links to other domains (example.com and example.net differ, but www.example.com and example.com do not) that are pure HTML
    * CSS

    Also not we need to be careful with how much we download!

    We don't want to sit and download an 8GB DVD or even an unlimited byte radio
    stream!

    We also want to avoid downloading the same page twice (once to get the links
    then again to scan the page itself).
    """
    response = requests.get(url, timeout=10.0)
    html = lxml.html.fromstring(
        response.content,
        base_url=response.url
    )
    html.make_links_absolute(resolve_base_href=True)
    links = {link for element, attribute, link, pos in html.iterlinks()}

    return map(clamd_scan, links)


def clamd_scan(url):
    return clamd_socket.scan_stream(requests.get(url, timeout=10.0).content)

if __name__ == "__main__":
    print clamd_scan("https://secure.eicar.org/eicar.com")

add and document niave html crawler, with future work