allowed_domains = ['dmoz.org']
start_urls = [
    'http://www.dmoz.org/Computers/Programming/Languages/Python/Books/',
    'http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/']
custom_settings = {'DOWNLOAD_DELAY': 5, 'CONCURRENT_REQUESTS_PER_DOMAIN': 2}
Запрос задержки Scrapy