# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


import zipfile
import cStringIO as StringIO
import csv
import redis
from urlparse import urlparse
from publicsuffix import PublicSuffixList
from celery import task
from celery.utils.functional import chunks
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS

    However it might be better to stick this in a *real* database local to
    each worker, and update it each day. Might as well use the Redis DB we
    have lying around. It might be even better to have a BloomFilter before
    even bothering to check the DB.
    """
    alexa_table = csv.DictReader(
        zipfile.ZipFile(
            StringIO.StringIO(
                requests.get("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip").content
            )
        ).open("top-1m.csv"),
        fieldnames=("score", "domain")
    )
    return {item["domain"]: item["score"] for item in alexa_table}


def update_alexa():
    pipe = redis_db["master"].pipeline()
    pipe.delete(domains_key)

    for chunk in dict_chunk(alexa_top_million(), 1000):
        pipe.hmset(domains_key, chunk)

    pipe.execute()


@task
def alexa_malware_scan(url):
    domain = PublicSuffixList().get_public_suffix(urlparse(url).netloc)  # IRIs are going to be a pain here.

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


# Path: malware_crawl/scan/alexa.py
import zipfile
import cStringIO as StringIO
import csv
import redis
from urlparse import urlparse
from publicsuffix import PublicSuffixList
from celery import task
from celery.utils.functional import chunks
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS


    However it might be better to stick this in a *real* database local to
    each worker, and update it each day. Might as well use the Redis DB we
    have lying around. It might be even better to have a BloomFilter before
    even bothering to check the DB.
    """
    alexa_table = csv.DictReader(
        zipfile.ZipFile(
            StringIO.StringIO(
                requests.get("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip").content
            )
        ).open("top-1m.csv"),
        fieldnames=("score", "domain")
    )
    return {item["domain"]: item["score"] for item in alexa_table}


def update_alexa():
    pipe = redis_db["master"].pipeline()
    pipe.delete(domains_key)

    for chunk in dict_chunk(alexa_top_million(), 1000):
        pipe.hmset(domains_key, chunk)

    pipe.execute()


@task
def alexa_malware_scan(url):
    domain = PublicSuffixList().get_public_suffix(urlparse(url).netloc)  # IRIs are going to be a pain here.

    pipe = redis_db["slave"].pipeline()