# Path: nyawc/http/Response.py
# class Response(object):
#     """Response placeholder class for before request is finished.
# 
#     Attributes:
#         url (str): The absolute URL of the request/response.
# 
#     Note:
#         This class will be replaced with the response class of Python's `requests` module when the
#         request is finished. For more information check http://docs.python-requests.org/en/master/api/#requests.Response.
# 
#     """
# 
#     def __init__(self, url):
#         """Constructs a Response instance.
# 
#         Args:
#             url (str): The absolute URL of the request/response.
# 
#         """
# 
#         self.url = url
# 
# Path: nyawc/QueueItem.py
# class QueueItem(object):
#     """The QueueItem class keeps track of the request and response and the crawling status.
# 
#     Attributes:
#         STATUS_QUEUED (str): Status for when the crawler did not yet start the request.
#         STATUS_IN_PROGRESS (str): Status for when the crawler is currently crawling the request.
#         STATUS_FINISHED (str): Status for when the crawler has finished crawling the request.
#         STATUS_CANCELLED (str): Status for when the crawler has cancelled the request.
#         STATUS_ERRORED (str): Status for when the crawler could not execute the request.
#         STATUSES (arr): All statuses.
#         status (str): The current crawling status.
#         decomposed (bool): If the this queue item is decomposed.
#         request (:class:`nyawc.http.Request`): The Request object.
#         response (:class:`nyawc.http.Response`): The Response object.
#         __response_soup (obj): The BeautifulSoup container for the response text.
#         __index_hash (str): The index of the queue (if cached), otherwise None.
# 
#     Note:
#         A queue item will be decomposed (cached objects are deleted to free up memory) when it is
#         not likeley to be used again. After decompisition variables will not be cached anymore.
# 
#     """
# 
#     STATUS_QUEUED = "queued"
# 
#     STATUS_IN_PROGRESS = "in_progress"
# 
#     STATUS_FINISHED = "finished"
# 
#     STATUS_CANCELLED = "cancelled"
# 
#     STATUS_ERRORED = "errored"
# 
#     STATUSES = [
#         STATUS_QUEUED,
#         STATUS_IN_PROGRESS,
#         STATUS_FINISHED,
#         STATUS_CANCELLED,
#         STATUS_ERRORED
#     ]
# 
#     def __init__(self, request, response):
#         """Constructs a QueueItem instance.
# 
#         Args:
#             request (:class:`nyawc.http.Request`): The Request object.
#             response (:class:`nyawc.http.Response`): The Response object (empty object when initialized).
# 
#         """
# 
#         self.status = QueueItem.STATUS_QUEUED
#         self.decomposed = False
#         self.__response_soup = None
#         self.__index_hash = None
# 
#         self.request = request
#         self.response = response
# 
#     def get_soup_response(self):
#         """Get the response as a cached BeautifulSoup container.
# 
#         Returns:
#             obj: The BeautifulSoup container.
# 
#         """
# 
#         if self.response is not None:
#             if self.__response_soup is None:
#                 result = BeautifulSoup(self.response.text, "lxml")
# 
#                 if self.decomposed:
#                     return result
#                 else:
#                     self.__response_soup = BeautifulSoup(self.response.text, "lxml")
# 
#         return self.__response_soup
# 
#     def decompose(self):
#         """Decompose this queue item (set cached variables to None) to free up memory.
# 
#         Note:
#             When setting cached variables to None memory will be released after the garbage 
#             collector ran.
#         
#         """
# 
#         self.__response_soup = None
# 
#         self.decomposed = True
# 
#     def get_hash(self):
#         """Generate and return the dict index hash of the given queue item.
# 
#         Note:
#             Cookies should not be included in the hash calculation because
#             otherwise requests are crawled multiple times with e.g. different
#             session keys, causing infinite crawling recursion.
# 
#         Note:
#             At this moment the keys do not actually get hashed since it works perfectly without and
#             since hashing the keys requires us to built hash collision management.
# 
#         Returns:
#             str: The hash of the given queue item.
# 
#         """
# 
#         if self.__index_hash:
#             return self.__index_hash
# 
#         key = self.request.method
# 
#         key += URLHelper.get_protocol(self.request.url)
#         key += URLHelper.get_subdomain(self.request.url)
#         key += URLHelper.get_hostname(self.request.url)
#         key += URLHelper.get_tld(self.request.url)
#         key += URLHelper.get_path(self.request.url)
# 
#         key += str(URLHelper.get_ordered_params(self.request.url))
# 
#         if self.request.data is not None:
#             key += str(self.request.data.keys())
# 
#         self.__index_hash = key
#         return self.__index_hash


from collections import OrderedDict
from nyawc.http.Response import Response
from nyawc.QueueItem import QueueItem
        self.items_finished = OrderedDict()
        self.items_cancelled = OrderedDict()
        self.items_errored = OrderedDict()

    def add_request(self, request):
        """Add a request to the queue.

        Args:
            request (:class:`nyawc.http.Request`): The request to add.

        Returns:
            :class:`nyawc.QueueItem`: The created queue item.

        """

        queue_item = QueueItem(request, Response(request.url))
        self.add(queue_item)
        return queue_item

    def has_request(self, request):
        """Check if the given request already exists in the queue.

        Args:
            request (:class:`nyawc.http.Request`): The request to check.

        Returns:
            bool: True if already exists, False otherwise.

        """


# Path: nyawc/http/Response.py
# class Response(object):
#     """Response placeholder class for before request is finished.
# 
#     Attributes:
#         url (str): The absolute URL of the request/response.
# 
#     Note:
#         This class will be replaced with the response class of Python's `requests` module when the
#         request is finished. For more information check http://docs.python-requests.org/en/master/api/#requests.Response.
# 
#     """
# 
#     def __init__(self, url):
#         """Constructs a Response instance.
# 
#         Args:
#             url (str): The absolute URL of the request/response.
# 
#         """
# 
#         self.url = url
# 
# Path: nyawc/QueueItem.py
# class QueueItem(object):
#     """The QueueItem class keeps track of the request and response and the crawling status.
# 
#     Attributes:
#         STATUS_QUEUED (str): Status for when the crawler did not yet start the request.
#         STATUS_IN_PROGRESS (str): Status for when the crawler is currently crawling the request.
#         STATUS_FINISHED (str): Status for when the crawler has finished crawling the request.
#         STATUS_CANCELLED (str): Status for when the crawler has cancelled the request.
#         STATUS_ERRORED (str): Status for when the crawler could not execute the request.
#         STATUSES (arr): All statuses.
#         status (str): The current crawling status.
#         decomposed (bool): If the this queue item is decomposed.
#         request (:class:`nyawc.http.Request`): The Request object.
#         response (:class:`nyawc.http.Response`): The Response object.
#         __response_soup (obj): The BeautifulSoup container for the response text.
#         __index_hash (str): The index of the queue (if cached), otherwise None.
# 
#     Note:
#         A queue item will be decomposed (cached objects are deleted to free up memory) when it is
#         not likeley to be used again. After decompisition variables will not be cached anymore.
# 
#     """
# 
#     STATUS_QUEUED = "queued"
# 
#     STATUS_IN_PROGRESS = "in_progress"
# 
#     STATUS_FINISHED = "finished"
# 
#     STATUS_CANCELLED = "cancelled"
# 
#     STATUS_ERRORED = "errored"
# 
#     STATUSES = [
#         STATUS_QUEUED,
#         STATUS_IN_PROGRESS,
#         STATUS_FINISHED,
#         STATUS_CANCELLED,
#         STATUS_ERRORED
#     ]
# 
#     def __init__(self, request, response):
#         """Constructs a QueueItem instance.
# 
#         Args:
#             request (:class:`nyawc.http.Request`): The Request object.
#             response (:class:`nyawc.http.Response`): The Response object (empty object when initialized).
# 
#         """
# 
#         self.status = QueueItem.STATUS_QUEUED
#         self.decomposed = False
#         self.__response_soup = None
#         self.__index_hash = None
# 
#         self.request = request
#         self.response = response
# 
#     def get_soup_response(self):
#         """Get the response as a cached BeautifulSoup container.
# 
#         Returns:
#             obj: The BeautifulSoup container.
# 
#         """
# 
#         if self.response is not None:
#             if self.__response_soup is None:
#                 result = BeautifulSoup(self.response.text, "lxml")
# 
#                 if self.decomposed:
#                     return result
#                 else:
#                     self.__response_soup = BeautifulSoup(self.response.text, "lxml")
# 
#         return self.__response_soup
# 
#     def decompose(self):
#         """Decompose this queue item (set cached variables to None) to free up memory.
# 
#         Note:
#             When setting cached variables to None memory will be released after the garbage 
#             collector ran.
#         
#         """
# 
#         self.__response_soup = None
# 
#         self.decomposed = True
# 
#     def get_hash(self):
#         """Generate and return the dict index hash of the given queue item.
# 
#         Note:
#             Cookies should not be included in the hash calculation because
#             otherwise requests are crawled multiple times with e.g. different
#             session keys, causing infinite crawling recursion.
# 
#         Note:
#             At this moment the keys do not actually get hashed since it works perfectly without and
#             since hashing the keys requires us to built hash collision management.
# 
#         Returns:
#             str: The hash of the given queue item.
# 
#         """
# 
#         if self.__index_hash:
#             return self.__index_hash
# 
#         key = self.request.method
# 
#         key += URLHelper.get_protocol(self.request.url)
#         key += URLHelper.get_subdomain(self.request.url)
#         key += URLHelper.get_hostname(self.request.url)
#         key += URLHelper.get_tld(self.request.url)
#         key += URLHelper.get_path(self.request.url)
# 
#         key += str(URLHelper.get_ordered_params(self.request.url))
# 
#         if self.request.data is not None:
#             key += str(self.request.data.keys())
# 
#         self.__index_hash = key
#         return self.__index_hash


# Path: nyawc/Queue.py
from collections import OrderedDict
from nyawc.http.Response import Response
from nyawc.QueueItem import QueueItem

        self.items_finished = OrderedDict()
        self.items_cancelled = OrderedDict()
        self.items_errored = OrderedDict()

    def add_request(self, request):
        """Add a request to the queue.

        Args:
            request (:class:`nyawc.http.Request`): The request to add.

        Returns:
            :class:`nyawc.QueueItem`: The created queue item.

        """

        queue_item = QueueItem(request, Response(request.url))
        self.add(queue_item)
        return queue_item

    def has_request(self, request):
        """Check if the given request already exists in the queue.

        Args:
            request (:class:`nyawc.http.Request`): The request to check.

        Returns:
            bool: True if already exists, False otherwise.

        """


        queue_item = QueueItem(request, Response(request.url))