# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }
# 
# Path: malware_crawl/models.py
# class CaptureUrl(models.Model):
#     url_id = models.BigIntegerField(unique=True, primary_key=True)
#     url = models.CharField(max_length=6249)
#     currentstatus = models.CharField(max_length=3, blank=True)
#     lastvisittime = models.CharField(max_length=69, blank=True)
#     operation_id = models.IntegerField(null=True, blank=True)
#     class Meta:
#         db_table = u'url'


from datetime import timedelta
from six.moves import zip, map
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db
from django.conf import settings
from malware_crawl.models import CaptureUrl
from malware_crawl.scan.celery_rpc_client import RevertRpcClient
import time
import cPickle as pickle
#    def route_for_task(self, task, args=None, kwargs=None):
#        if task == 'malware_crawl.scan.capture_hpc.chpc_malware_scan':
#            return {'queue': 'capturehpc',
#                'routing_key': 'capturehpc.scan'}
#        return None


def redis_cache(key_prefix, prepare, request, items):
    items = list(map(prepare, items))
    unique_items = list(set(items))
    total_response = {}

    # get all the items that are already in cache
    #pipe = redis_db["slave"].pipeline()
    #for item in unique_items:
    #    pipe.get(key_prefix.format(item=item))

    cache_misses = set()
    # add those items we already know about to the total_response
    #for item, cache in zip(unique_items, pipe.execute()):
    #    if cache is not None:
    #        total_response[item] = pickle.loads(cache)
    #    else:
    #        cache_misses.add(item)
    cache_misses = set(unique_items)

    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET

# Path: malware_crawl/defaults.py
# REQUESTS_SESSIONS = {
#     "api": requests.Session(),
#     "probe": requests.Session()
# }
# 
# Path: malware_crawl/models.py
# class CaptureUrl(models.Model):
#     url_id = models.BigIntegerField(unique=True, primary_key=True)
#     url = models.CharField(max_length=6249)
#     currentstatus = models.CharField(max_length=3, blank=True)
#     lastvisittime = models.CharField(max_length=69, blank=True)
#     operation_id = models.IntegerField(null=True, blank=True)
#     class Meta:
#         db_table = u'url'


# Path: malware_crawl/scan/capture_hpc.py
from datetime import timedelta
from six.moves import zip, map
from celery import task, current_app
from celery.contrib.batches import Batches
from malware_crawl.defaults import redis_db
from django.conf import settings
from malware_crawl.models import CaptureUrl
from malware_crawl.scan.celery_rpc_client import RevertRpcClient
import time
import cPickle as pickle

#    def route_for_task(self, task, args=None, kwargs=None):
#        if task == 'malware_crawl.scan.capture_hpc.chpc_malware_scan':
#            return {'queue': 'capturehpc',
#                'routing_key': 'capturehpc.scan'}
#        return None


def redis_cache(key_prefix, prepare, request, items):
    items = list(map(prepare, items))
    unique_items = list(set(items))
    total_response = {}

    # get all the items that are already in cache
    #pipe = redis_db["slave"].pipeline()
    #for item in unique_items:
    #    pipe.get(key_prefix.format(item=item))

    cache_misses = set()
    # add those items we already know about to the total_response
    #for item, cache in zip(unique_items, pipe.execute()):
    #    if cache is not None:
    #        total_response[item] = pickle.loads(cache)
    #    else:
    #        cache_misses.add(item)
    cache_misses = set(unique_items)

    # use the given function to collect the missing values
    total_response.update(request(cache_misses))

    # add to the cache the result from the INTERNET

    pipe = redis_db["master"].pipeline()