parameters: (self, link, referrer, rel)
return_statement: return result
docstring: Determine whether a link URL from a referring page and with a
  
          particular "rel" attribute should be queued for scraping.
function: def _should_queue(self, link, referrer, rel):
  
          """
  
          Determine whether a link URL from a referring page and with a
  
          particular "rel" attribute should be queued for scraping.
  
          """
  
          scheme, netloc, path, _, _, _ = urlparse(link)
  
          if path.endswith(self.source_extensions + self.binary_extensions +
  
                           self.excluded_extensions):
  
              result = False
  
          elif self.skip_externals and not link.startswith(self.base_url):
  
              result = False
  
          elif not referrer.startswith(self.base_url):
  
              result = False
  
          elif rel not in ('homepage', 'download'):
  
              result = False
  
          elif scheme not in ('http', 'https', 'ftp'):
  
              result = False
  
          elif self._is_platform_dependent(link):
  
              result = False
  
          else:
  
              host = netloc.split(':', 1)[0]
  
              if host.lower() == 'localhost':
  
                  result = False
  
              else:
  
                  result = True
  
          logger.debug('should_queue: %s (%s) from %s -> %s', link, rel,
  
                       referrer, result)
  
          return result
function_tokens: def _should_queue ( self , link , referrer , rel ) : scheme , netloc , path , _ , _ , _ = urlparse ( link ) if path . endswith ( self . source_extensions + self . binary_extensions + self . excluded_extensions ) : result = False elif self . skip_externals and not link . startswith ( self . base_url ) : result = False elif not referrer . startswith ( self . base_url ) : result = False elif rel not in ( 'homepage' , 'download' ) : result = False elif scheme not in ( 'http' , 'https' , 'ftp' ) : result = False elif self . _is_platform_dependent ( link ) : result = False else : host = netloc . split ( ':' , 1 ) [ 0 ] if host . lower ( ) == 'localhost' : result = False else : result = True logger . debug ( 'should_queue: %s (%s) from %s -> %s' , link , rel , referrer , result ) return result