# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


from urlparse import urlparse
from datetime import timedelta
from six.moves import zip, map
from six import binary_type
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS
from django.conf import settings
import cPickle as pickle


requests = REQUESTS_SESSIONS["api"]


wot_api_target = "https://api.mywot.com/0.4/public_link_json"


def redis_cache(key_prefix, prepare, request, items):
    items = list(map(prepare, items))
    unique_items = list(set(items))
    total_response = {}

    # get all the items that are already in cache
    pipe = redis_db["slave"].pipeline()
    for item in unique_items:
        pipe.get(key_prefix.format(item=item))

    cache_misses = set()
    # add those items we already know about to the total_response
    for item, cache in zip(unique_items, pipe.execute()):
        if cache is not None:
            total_response[item] = pickle.loads(cache)
        else:
            cache_misses.add(item)

    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }


# Path: malware_crawl/scan/web_api.py
from urlparse import urlparse
from datetime import timedelta
from six.moves import zip, map
from six import binary_type
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db, REQUESTS_SESSIONS
from django.conf import settings
import cPickle as pickle



requests = REQUESTS_SESSIONS["api"]


wot_api_target = "https://api.mywot.com/0.4/public_link_json"


def redis_cache(key_prefix, prepare, request, items):
    items = list(map(prepare, items))
    unique_items = list(set(items))
    total_response = {}

    # get all the items that are already in cache
    pipe = redis_db["slave"].pipeline()
    for item in unique_items:
        pipe.get(key_prefix.format(item=item))

    cache_misses = set()
    # add those items we already know about to the total_response
    for item, cache in zip(unique_items, pipe.execute()):
        if cache is not None:
            total_response[item] = pickle.loads(cache)
        else:
            cache_misses.add(item)

    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET

    pipe = redis_db["master"].pipeline()