def process_spider_output(self, response, result, spider):

    def _filter(request):
        if isinstance(request, Request):
            domain = tldextract.extract(request.url).registered_domain
maxdepth = self.domain_depths.get(domain, self.default_depth)
depth = response.meta.get('depth', 0) + 1
request.meta['depth'] = depth
if maxdepth and depth > maxdepth:
    log.msg(format='Ignoring link (depth > %(maxdepth)d): %(requrl)s ',
        level=log.DEBUG, spider=spider, maxdepth=maxdepth, requrl=request.url)
return False
return True
return (r for r in result or () if _filter(r))
Scrapy устанавливает предел глубины для разрешённых_доменов