parameters: (self)
docstring: Reads the robots.txt URL and feeds it to the parser.
docstring_summary: Reads the robots.txt URL and feeds it to the parser.
docstring_tokens: Reads the robots . txt URL and feeds it to the parser .
function: def read(self):
          """Reads the robots.txt URL and feeds it to the parser."""
          opener = URLopener()
          f = opener.open(self.url)
          lines = [line.strip() for line in f]
          f.close()
          self.errcode = opener.errcode
          if self.errcode in (401, 403):
              self.disallow_all = True
          elif self.errcode >= 400 and self.errcode < 500:
              self.allow_all = True
          elif self.errcode == 200 and lines:
              self.parse(lines)
function_tokens: def read ( self ) : opener = URLopener ( ) f = opener . open ( self . url ) lines = [ line . strip ( ) for line in f ] f . close ( ) self . errcode = opener . errcode if self . errcode in ( 401 , 403 ) : self . disallow_all = True elif self . errcode >= 400 and self . errcode < 500 : self . allow_all = True elif self . errcode == 200 and lines : self . parse ( lines )