# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


from urlparse import urlparse
from datetime import timedelta
from six.moves import zip, map
from six import binary_type
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS
from django.conf import settings
import cPickle as pickle

    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET
    pipe = redis_db["master"].pipeline()
    cache_length = timedelta(minutes=30).seconds
    for item in cache_misses:
        pipe.setex(key_prefix.format(item=item),
            pickle.dumps(total_response[item], -1),
            cache_length
        )
    pipe.execute()

    return [total_response[item] for item in items]


@task(base=Batches, flush_every=100, flush_interval=((10, 0.1)[settings.DEBUG]))
def wot_malware_scan(requests):
    sig = lambda url: url
    reponses = wot_malware_scan_real(
        (sig(*request.args, **request.kwargs) for request in requests)
    )
    # use mark_as_done to manually return response data
    for response, request in zip(reponses, requests):
        current_app.backend.mark_as_done(request.id, response)


def prepare_requests(domains):
    "ensure requests to the WOT api are less than 8KiB"

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


# Path: malware_crawl/scan/web_api.py
from urlparse import urlparse
from datetime import timedelta
from six.moves import zip, map
from six import binary_type
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS
from django.conf import settings
import cPickle as pickle


    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET
    pipe = redis_db["master"].pipeline()
    cache_length = timedelta(minutes=30).seconds
    for item in cache_misses:
        pipe.setex(key_prefix.format(item=item),
            pickle.dumps(total_response[item], -1),
            cache_length
        )
    pipe.execute()

    return [total_response[item] for item in items]


@task(base=Batches, flush_every=100, flush_interval=((10, 0.1)[settings.DEBUG]))
def wot_malware_scan(requests):
    sig = lambda url: url
    reponses = wot_malware_scan_real(
        (sig(*request.args, **request.kwargs) for request in requests)
    )
    # use mark_as_done to manually return response data
    for response, request in zip(reponses, requests):
        current_app.backend.mark_as_done(request.id, response)


def prepare_requests(domains):
    "ensure requests to the WOT api are less than 8KiB"

    prepared = list(map(lambda domain: binary_type(domain + '/'), set(domains)))