# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


import zipfile
import cStringIO as StringIO
import csv
import redis
from urlparse import urlparse
from publicsuffix import PublicSuffixList
from celery import task
from celery.utils.functional import chunks
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS
    for chunk in chunks(items, chunk_size):
        yield dict(chunk)


def alexa_top_million():
    """Returns an iterable of objects describing the top million domains from Alexa
    It turns out this list can be used as a set and held in memory fairly
    easily:

    domains = frozenset({item["domain"] for item in alexa_top_million()})
    if "google.com" in domains:
        return u"safe"

    However it might be better to stick this in a *real* database local to
    each worker, and update it each day. Might as well use the Redis DB we
    have lying around. It might be even better to have a BloomFilter before
    even bothering to check the DB.
    """
    alexa_table = csv.DictReader(
        zipfile.ZipFile(
            StringIO.StringIO(
                requests.get("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip").content
            )
        ).open("top-1m.csv"),
        fieldnames=("score", "domain")
    )
    return {item["domain"]: item["score"] for item in alexa_table}


def update_alexa():

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


# Path: malware_crawl/scan/alexa.py
import zipfile
import cStringIO as StringIO
import csv
import redis
from urlparse import urlparse
from publicsuffix import PublicSuffixList
from celery import task
from celery.utils.functional import chunks
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS

    for chunk in chunks(items, chunk_size):
        yield dict(chunk)


def alexa_top_million():
    """Returns an iterable of objects describing the top million domains from Alexa
    It turns out this list can be used as a set and held in memory fairly
    easily:

    domains = frozenset({item["domain"] for item in alexa_top_million()})
    if "google.com" in domains:
        return u"safe"

    However it might be better to stick this in a *real* database local to
    each worker, and update it each day. Might as well use the Redis DB we
    have lying around. It might be even better to have a BloomFilter before
    even bothering to check the DB.
    """
    alexa_table = csv.DictReader(
        zipfile.ZipFile(
            StringIO.StringIO(
                requests.get("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip").content
            )
        ).open("top-1m.csv"),
        fieldnames=("score", "domain")
    )
    return {item["domain"]: item["score"] for item in alexa_table}


def update_alexa():

    pipe = redis_db["master"].pipeline()