def parse_page(self, response):
    """ Scrape useful stuff from page, and spawn new requests
    """
hxs = HtmlXPathSelector(response)
images = hxs.select('//img')
links = hxs.select('//a/@href')
for link in links:
    yield Request(url=link, callback=self.parse_page)
Using Scrapy with authenticated (logged in) user session