parameters: (self)
docstring: Get a URL to fetch from the work queue, get the HTML page, examine its
          links for download candidates and candidates for further scraping.
  
          This is a handy method to run in a thread.
docstring_summary: Get a URL to fetch from the work queue, get the HTML page, examine its
          links for download candidates and candidates for further scraping.
docstring_tokens: Get a URL to fetch from the work queue get the HTML page examine its links for download candidates and candidates for further scraping .
function: def _fetch(self):
          """
          Get a URL to fetch from the work queue, get the HTML page, examine its
          links for download candidates and candidates for further scraping.
  
          This is a handy method to run in a thread.
          """
          while True:
              url = self._to_fetch.get()
              try:
                  if url:
                      page = self.get_page(url)
                      if page is None:    # e.g. after an error
                          continue
                      for link, rel in page.links:
                          if link not in self._seen:
                              self._seen.add(link)
                              if (not self._process_download(link) and
                                  self._should_queue(link, url, rel)):
                                  logger.debug('Queueing %s from %s', link, url)
                                  self._to_fetch.put(link)
              finally:
                  # always do this, to avoid hangs :-)
                  self._to_fetch.task_done()
              if not url:
                  #logger.debug('Sentinel seen, quitting.')
                  break
function_tokens: def _fetch ( self ) : while True : url = self . _to_fetch . get ( ) try : if url : page = self . get_page ( url ) if page is None : # e.g. after an error continue for link , rel in page . links : if link not in self . _seen : self . _seen . add ( link ) if ( not self . _process_download ( link ) and self . _should_queue ( link , url , rel ) ) : logger . debug ( 'Queueing %s from %s' , link , url ) self . _to_fetch . put ( link ) finally : # always do this, to avoid hangs :-) self . _to_fetch . task_done ( ) if not url : #logger.debug('Sentinel seen, quitting.') break