def main_task(reactor):

    def fetch_urls():
        for url in get_urls():
            yield getPage(url).addCallback(parse)
coop = task.Cooperator()
urls = fetch_urls()
return defer.DeferredList([coop.coiterate(urls) for _ in range(BATCH_SIZE)]
    ).addCallback(task_finished)
task.react(main_task)
Twisted: Waiting for subtasks to finish