# Path: thezombies/tasks/utils.py
# COUNTDOWN_MODULO = 21
# class ResultDict(dict):
# class InsecureHttpAdapter(HTTPAdapter):
#     def __init__(self, data=None, errors=None):
#     def add_error(self, error):
#     def errors(self):
# def response_to_dict(resp, history=True):
# def error_handler(uuid):
#     def init_poolmanager(self, connections, maxsize, block=False):
# 
# Path: thezombies/models.py
# class URLInspection(models.Model):
#     created_at = models.DateTimeField(auto_now_add=True)
#     url = models.TextField(blank=True, null=True)  # We may get (and want to store) really long or invalid urls, so...
#     requested_url = models.TextField()  # We may get (and want to store) really long or invalid urls, so...
#     encoding = models.CharField(max_length=120, blank=True, null=True)
#     apparent_encoding = models.CharField(max_length=120, blank=True, null=True)
#     content = models.OneToOneField(ResponseContent, null=True, related_name='content_for', editable=False)
#     history = hstore.ReferencesField(blank=True, null=True)
#     parent = models.ForeignKey('self', blank=True, null=True, on_delete=models.SET_NULL)
#     status_code = models.IntegerField(max_length=3, blank=True, null=True)
#     reason = models.CharField(blank=True, null=True, max_length=80, help_text='Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".')
#     headers = hstore.DictionaryField(default=dictionary_default)
#     timeout = models.BooleanField(default=False)
#     probe = models.ForeignKey('Probe', null=True, blank=True, related_name='url_inspections')
# 
#     objects = URLInspectionManager.from_queryset(URLInspectionQuerySet)()
# 
#     class Meta:
#         verbose_name = 'URL Inspection'
#         verbose_name_plural = 'URL Inspections'
#         get_latest_by = 'created_at'
#         ordering = ('-created_at', 'requested_url')
# 
#     def __repr__(self):
#         return '<URLInspection: {0} : {1}>'.format(self.requested_url, self.status_code)
# 
#     def __str__(self):
#         return self.__repr__()
# 
# class Probe(models.Model):
#     """A component of an Audit that takes some initial data and
#     stores a result of some tasks performed on that data
#     """
# 
#     GENERIC_PROBE = 0
#     URL_PROBE = 1
#     JSON_PROBE = 2
#     VALIDATION_PROBE = 3
# 
#     PROBE_TYPE_CHOICES = (
#         (GENERIC_PROBE, 'Generic Probe'),
#         (URL_PROBE, 'URL Probe'),
#         (JSON_PROBE, 'JSON Probe'),
#         (VALIDATION_PROBE, 'Validation Probe'),
#     )
# 
#     created_at = models.DateTimeField(auto_now_add=True)
#     updated_at = models.DateTimeField(auto_now=True)
#     probe_type = models.PositiveSmallIntegerField(choices=PROBE_TYPE_CHOICES, default=GENERIC_PROBE)
#     previous = models.ForeignKey('self', related_name='next', blank=True, null=True, on_delete=models.SET_NULL)
#     initial = hstore.DictionaryField(blank=True, null=True, default=dictionary_default)
#     result = hstore.DictionaryField(blank=True, null=True, default=dictionary_default)
#     errors = TextArrayField(blank=True, null=True, default=list_default)
#     audit = models.ForeignKey('Audit', null=True, blank=True)
# 
#     objects = ProbeQuerySet.as_manager()
# 
#     def __repr__(self):
#         return '<{0}: {1}>'.format(self.get_probe_type_display(), self.id)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     class Meta:
#         get_latest_by = 'created_at'
#         ordering = ('-created_at',)
# 
#     def error_count(self):
#         return len(self.errors)
# 
#     def get_absolute_url(self):
#         return reverse('probe-detail', kwargs={'pk': str(self.pk)})


from django.db import transaction
from django.conf import settings
from django_atomic_celery import task
from requests.exceptions import InvalidURL
from .utils import (ResultDict, logger, response_to_dict, InsecureHttpAdapter)
from thezombies.models import URLInspection, Probe
    from urllib.parse import urlparse, urlunparse
    from urlparse import urlparse, urlunparse
import requests
        except Exception as e:
            returnval.add_error(e)
        returnval['url_request_attempted'] = True
        # a non-None requests.Response will evaluate to False if it carries an HTTPError value
        if resp is not None:
            try:
                resp.raise_for_status()
            except Exception as e:
                returnval.add_error(e)
            if isinstance(resp, requests.Response):
                returnval['response'] = response_to_dict(resp)
            else:
                logger.error('session.request did not return a valid Response object')
    logger.info('Returning from request_url')
    return returnval


@task
def inspect_url(taskarg):
    """Task to check a URL and store some information about it. Tracks and returns errors.

    :param taskarg: A dictionary containing a url, and optionally a audit_id
    """
    returnval = ResultDict(taskarg)
    url = taskarg.get('url', None)
    url_type = taskarg.get('url_type', None)
    audit_id = taskarg.get('audit_id', None)
    prev_probe_id = taskarg.get('prev_probe_id', None)
    probe = None
    with transaction.atomic():

# Path: thezombies/tasks/utils.py
# COUNTDOWN_MODULO = 21
# class ResultDict(dict):
# class InsecureHttpAdapter(HTTPAdapter):
#     def __init__(self, data=None, errors=None):
#     def add_error(self, error):
#     def errors(self):
# def response_to_dict(resp, history=True):
# def error_handler(uuid):
#     def init_poolmanager(self, connections, maxsize, block=False):
# 
# Path: thezombies/models.py
# class URLInspection(models.Model):
#     created_at = models.DateTimeField(auto_now_add=True)
#     url = models.TextField(blank=True, null=True)  # We may get (and want to store) really long or invalid urls, so...
#     requested_url = models.TextField()  # We may get (and want to store) really long or invalid urls, so...
#     encoding = models.CharField(max_length=120, blank=True, null=True)
#     apparent_encoding = models.CharField(max_length=120, blank=True, null=True)
#     content = models.OneToOneField(ResponseContent, null=True, related_name='content_for', editable=False)
#     history = hstore.ReferencesField(blank=True, null=True)
#     parent = models.ForeignKey('self', blank=True, null=True, on_delete=models.SET_NULL)
#     status_code = models.IntegerField(max_length=3, blank=True, null=True)
#     reason = models.CharField(blank=True, null=True, max_length=80, help_text='Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".')
#     headers = hstore.DictionaryField(default=dictionary_default)
#     timeout = models.BooleanField(default=False)
#     probe = models.ForeignKey('Probe', null=True, blank=True, related_name='url_inspections')
# 
#     objects = URLInspectionManager.from_queryset(URLInspectionQuerySet)()
# 
#     class Meta:
#         verbose_name = 'URL Inspection'
#         verbose_name_plural = 'URL Inspections'
#         get_latest_by = 'created_at'
#         ordering = ('-created_at', 'requested_url')
# 
#     def __repr__(self):
#         return '<URLInspection: {0} : {1}>'.format(self.requested_url, self.status_code)
# 
#     def __str__(self):
#         return self.__repr__()
# 
# class Probe(models.Model):
#     """A component of an Audit that takes some initial data and
#     stores a result of some tasks performed on that data
#     """
# 
#     GENERIC_PROBE = 0
#     URL_PROBE = 1
#     JSON_PROBE = 2
#     VALIDATION_PROBE = 3
# 
#     PROBE_TYPE_CHOICES = (
#         (GENERIC_PROBE, 'Generic Probe'),
#         (URL_PROBE, 'URL Probe'),
#         (JSON_PROBE, 'JSON Probe'),
#         (VALIDATION_PROBE, 'Validation Probe'),
#     )
# 
#     created_at = models.DateTimeField(auto_now_add=True)
#     updated_at = models.DateTimeField(auto_now=True)
#     probe_type = models.PositiveSmallIntegerField(choices=PROBE_TYPE_CHOICES, default=GENERIC_PROBE)
#     previous = models.ForeignKey('self', related_name='next', blank=True, null=True, on_delete=models.SET_NULL)
#     initial = hstore.DictionaryField(blank=True, null=True, default=dictionary_default)
#     result = hstore.DictionaryField(blank=True, null=True, default=dictionary_default)
#     errors = TextArrayField(blank=True, null=True, default=list_default)
#     audit = models.ForeignKey('Audit', null=True, blank=True)
# 
#     objects = ProbeQuerySet.as_manager()
# 
#     def __repr__(self):
#         return '<{0}: {1}>'.format(self.get_probe_type_display(), self.id)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     class Meta:
#         get_latest_by = 'created_at'
#         ordering = ('-created_at',)
# 
#     def error_count(self):
#         return len(self.errors)
# 
#     def get_absolute_url(self):
#         return reverse('probe-detail', kwargs={'pk': str(self.pk)})


# Path: thezombies/tasks/urls.py
from django.db import transaction
from django.conf import settings
from django_atomic_celery import task
from requests.exceptions import InvalidURL
from .utils import (ResultDict, logger, response_to_dict, InsecureHttpAdapter)
from thezombies.models import URLInspection, Probe
    from urllib.parse import urlparse, urlunparse
    from urlparse import urlparse, urlunparse
import requests

        except Exception as e:
            returnval.add_error(e)
        returnval['url_request_attempted'] = True
        # a non-None requests.Response will evaluate to False if it carries an HTTPError value
        if resp is not None:
            try:
                resp.raise_for_status()
            except Exception as e:
                returnval.add_error(e)
            if isinstance(resp, requests.Response):
                returnval['response'] = response_to_dict(resp)
            else:
                logger.error('session.request did not return a valid Response object')
    logger.info('Returning from request_url')
    return returnval


@task
def inspect_url(taskarg):
    """Task to check a URL and store some information about it. Tracks and returns errors.

    :param taskarg: A dictionary containing a url, and optionally a audit_id
    """
    returnval = ResultDict(taskarg)
    url = taskarg.get('url', None)
    url_type = taskarg.get('url_type', None)
    audit_id = taskarg.get('audit_id', None)
    prev_probe_id = taskarg.get('prev_probe_id', None)
    probe = None
    with transaction.atomic():

        probe = Probe.objects.create(probe_type=Probe.URL_PROBE,