def _requests_to_follow(self, response):
    seen = set()
    for (n, rule) in enumerate(self._rules):
        links = [lnk for lnk in rule.link_extractor.extract_links(response) if (lnk not in seen)]
        if (links and rule.process_links):
            links = rule.process_links(links)
        for link in links:
            seen.add(link)
            r = self._build_request(n, link)
            (yield rule.process_request(r))
def _requests_to_follow(self, response):
    seen = set()
    for (n, rule) in enumerate(self._rules):
        links = [lnk for lnk in rule.link_extractor.extract_links(response) if (lnk not in seen)]
        if (links and rule.process_links):
            links = rule.process_links(links)
        for link in links:
            seen.add(link)
            r = self._build_request(n, link)
            (yield rule.process_request(r))