# Path: xpaw/http.py
# class HttpRequest:
#     def __init__(self, url, method="GET", body=None, params=None, headers=None, proxy=None,
#                  timeout=20, verify_ssl=False, allow_redirects=True, auth=None, proxy_auth=None,
#                  priority=None, dont_filter=False, callback=None, errback=None, meta=None,
#                  render=None):
#         """
#         Construct an HTTP request.
#         """
#         self.url = make_url(url, params)
#         self.method = method
#         self.body = body
#         self.headers = headers
#         self.proxy = proxy
#         self.timeout = timeout
#         self.verify_ssl = verify_ssl
#         self.allow_redirects = allow_redirects
#         self.auth = auth
#         self.proxy_auth = proxy_auth
#         self.priority = priority
#         self.dont_filter = dont_filter
#         self.callback = callback
#         self.errback = errback
#         self._meta = dict(meta) if meta else {}
#         self.render = render
# 
#     def __str__(self):
#         return '<{}, {}>'.format(self.method, self.url)
# 
#     __repr__ = __str__
# 
#     @property
#     def meta(self):
#         return self._meta
# 
#     def copy(self):
#         return self.replace()
# 
#     def replace(self, **kwargs):
#         for i in ["url", "method", "body", "headers", "proxy",
#                   "timeout", "verify_ssl", "allow_redirects", "auth", "proxy_auth",
#                   "priority", "dont_filter", "callback", "errback", "meta",
#                   "render"]:
#             kwargs.setdefault(i, getattr(self, i))
#         return type(self)(**kwargs)
# 
#     def to_dict(self):
#         callback = self.callback
#         if inspect.ismethod(callback):
#             callback = callback.__name__
#         errback = self.errback
#         if inspect.ismethod(errback):
#             errback = errback.__name__
#         d = {
#             'url': self.url,
#             'method': self.method,
#             'body': self.body,
#             'headers': self.headers,
#             'proxy': self.proxy,
#             'timeout': self.timeout,
#             'verify_ssl': self.verify_ssl,
#             'allow_redirects': self.allow_redirects,
#             'auth': self.auth,
#             'proxy_auth': self.proxy_auth,
#             'priority': self.priority,
#             'dont_filter': self.dont_filter,
#             'callback': callback,
#             'errback': errback,
#             'meta': self.meta,
#             'render': self.render
#         }
#         return d
# 
#     @classmethod
#     def from_dict(cls, d):
#         return cls(**d)


import logging
from xpaw.http import HttpRequest


class DepthMiddleware:
    def __init__(self, max_depth=None):
        self._max_depth = max_depth

    def __repr__(self):
        cls_name = self.__class__.__name__
        return '{}(max_depth={})'.format(cls_name, repr(self._max_depth))

    @classmethod
    def from_crawler(cls, crawler):
        config = crawler.config
        max_depth = config.getint('max_depth')
        return cls(max_depth=max_depth)

    def handle_spider_output(self, response, result):
        depth = response.meta.get('depth', 0) + 1
        for r in result:
            if isinstance(r, HttpRequest):
                r.meta['depth'] = depth
                if self._max_depth is None or depth <= self._max_depth:
                    yield r
                else:
                    log.debug("Abort %s: depth > %s", r, self._max_depth)
            else:
                yield r

    def handle_start_requests(self, result):
        for r in result:

# Path: xpaw/http.py
# class HttpRequest:
#     def __init__(self, url, method="GET", body=None, params=None, headers=None, proxy=None,
#                  timeout=20, verify_ssl=False, allow_redirects=True, auth=None, proxy_auth=None,
#                  priority=None, dont_filter=False, callback=None, errback=None, meta=None,
#                  render=None):
#         """
#         Construct an HTTP request.
#         """
#         self.url = make_url(url, params)
#         self.method = method
#         self.body = body
#         self.headers = headers
#         self.proxy = proxy
#         self.timeout = timeout
#         self.verify_ssl = verify_ssl
#         self.allow_redirects = allow_redirects
#         self.auth = auth
#         self.proxy_auth = proxy_auth
#         self.priority = priority
#         self.dont_filter = dont_filter
#         self.callback = callback
#         self.errback = errback
#         self._meta = dict(meta) if meta else {}
#         self.render = render
# 
#     def __str__(self):
#         return '<{}, {}>'.format(self.method, self.url)
# 
#     __repr__ = __str__
# 
#     @property
#     def meta(self):
#         return self._meta
# 
#     def copy(self):
#         return self.replace()
# 
#     def replace(self, **kwargs):
#         for i in ["url", "method", "body", "headers", "proxy",
#                   "timeout", "verify_ssl", "allow_redirects", "auth", "proxy_auth",
#                   "priority", "dont_filter", "callback", "errback", "meta",
#                   "render"]:
#             kwargs.setdefault(i, getattr(self, i))
#         return type(self)(**kwargs)
# 
#     def to_dict(self):
#         callback = self.callback
#         if inspect.ismethod(callback):
#             callback = callback.__name__
#         errback = self.errback
#         if inspect.ismethod(errback):
#             errback = errback.__name__
#         d = {
#             'url': self.url,
#             'method': self.method,
#             'body': self.body,
#             'headers': self.headers,
#             'proxy': self.proxy,
#             'timeout': self.timeout,
#             'verify_ssl': self.verify_ssl,
#             'allow_redirects': self.allow_redirects,
#             'auth': self.auth,
#             'proxy_auth': self.proxy_auth,
#             'priority': self.priority,
#             'dont_filter': self.dont_filter,
#             'callback': callback,
#             'errback': errback,
#             'meta': self.meta,
#             'render': self.render
#         }
#         return d
# 
#     @classmethod
#     def from_dict(cls, d):
#         return cls(**d)


# Path: xpaw/extensions/depth.py
import logging
from xpaw.http import HttpRequest



class DepthMiddleware:
    def __init__(self, max_depth=None):
        self._max_depth = max_depth

    def __repr__(self):
        cls_name = self.__class__.__name__
        return '{}(max_depth={})'.format(cls_name, repr(self._max_depth))

    @classmethod
    def from_crawler(cls, crawler):
        config = crawler.config
        max_depth = config.getint('max_depth')
        return cls(max_depth=max_depth)

    def handle_spider_output(self, response, result):
        depth = response.meta.get('depth', 0) + 1
        for r in result:
            if isinstance(r, HttpRequest):
                r.meta['depth'] = depth
                if self._max_depth is None or depth <= self._max_depth:
                    yield r
                else:
                    log.debug("Abort %s: depth > %s", r, self._max_depth)
            else:
                yield r

    def handle_start_requests(self, result):
        for r in result:

            if isinstance(r, HttpRequest):