parameters: (self, useragent, url)
return_statement: return True
docstring: using the parsed robots.txt decide if useragent can fetch url
docstring_summary: using the parsed robots.txt decide if useragent can fetch url
docstring_tokens: using the parsed robots . txt decide if useragent can fetch url
function: def can_fetch(self, useragent, url):
          """using the parsed robots.txt decide if useragent can fetch url"""
          if self.disallow_all:
              return False
          if self.allow_all:
              return True
  
          # Until the robots.txt file has been read or found not
          # to exist, we must assume that no url is allowable.
          # This prevents false positives when a user erroneously
          # calls can_fetch() before calling read().
          if not self.last_checked:
              return False
  
          # search for given user agent matches
          # the first match counts
          parsed_url = urlparse.urlparse(urllib.unquote(url))
          url = urlparse.urlunparse(('', '', parsed_url.path,
              parsed_url.params, parsed_url.query, parsed_url.fragment))
          url = urllib.quote(url)
          if not url:
              url = "/"
          for entry in self.entries:
              if entry.applies_to(useragent):
                  return entry.allowance(url)
          # try the default entry last
          if self.default_entry:
              return self.default_entry.allowance(url)
          # agent not found ==> access granted
          return True
function_tokens: def can_fetch ( self , useragent , url ) : if self . disallow_all : return False if self . allow_all : return True # Until the robots.txt file has been read or found not # to exist, we must assume that no url is allowable. # This prevents false positives when a user erroneously # calls can_fetch() before calling read(). if not self . last_checked : return False # search for given user agent matches # the first match counts parsed_url = urlparse . urlparse ( urllib . unquote ( url ) ) url = urlparse . urlunparse ( ( '' , '' , parsed_url . path , parsed_url . params , parsed_url . query , parsed_url . fragment ) ) url = urllib . quote ( url ) if not url : url = "/" for entry in self . entries : if entry . applies_to ( useragent ) : return entry . allowance ( url ) # try the default entry last if self . default_entry : return self . default_entry . allowance ( url ) # agent not found ==> access granted return True