rules = Rule(LxmlLinkExtractor(allow=()), callback='parse_obj', follow=True),

def parse_obj(self, response):
    for link in LxmlLinkExtractor(allow=(), deny=self.allowed_domains
        ).extract_links(response):
        item = someItem()
item['url'] = link.url
Scrapy, only follow internal URLS but extract all links found