parameters: (self, lines)
docstring: parse the input lines from a robots.txt file.
             We allow that a user-agent: line is not preceded by
             one or more blank lines.
docstring_summary: parse the input lines from a robots.txt file.
             We allow that a user-agent: line is not preceded by
             one or more blank lines.
docstring_tokens: parse the input lines from a robots . txt file . We allow that a user - agent : line is not preceded by one or more blank lines .
function: def parse(self, lines):
          """parse the input lines from a robots.txt file.
             We allow that a user-agent: line is not preceded by
             one or more blank lines."""
          # states:
          #   0: start state
          #   1: saw user-agent line
          #   2: saw an allow or disallow line
          state = 0
          linenumber = 0
          entry = Entry()
  
          self.modified()
          for line in lines:
              linenumber += 1
              if not line:
                  if state == 1:
                      entry = Entry()
                      state = 0
                  elif state == 2:
                      self._add_entry(entry)
                      entry = Entry()
                      state = 0
              # remove optional comment and strip line
              i = line.find('#')
              if i >= 0:
                  line = line[:i]
              line = line.strip()
              if not line:
                  continue
              line = line.split(':', 1)
              if len(line) == 2:
                  line[0] = line[0].strip().lower()
                  line[1] = urllib.unquote(line[1].strip())
                  if line[0] == "user-agent":
                      if state == 2:
                          self._add_entry(entry)
                          entry = Entry()
                      entry.useragents.append(line[1])
                      state = 1
                  elif line[0] == "disallow":
                      if state != 0:
                          entry.rulelines.append(RuleLine(line[1], False))
                          state = 2
                  elif line[0] == "allow":
                      if state != 0:
                          entry.rulelines.append(RuleLine(line[1], True))
                          state = 2
          if state == 2:
              self._add_entry(entry)
function_tokens: def parse ( self , lines ) : # states: #   0: start state #   1: saw user-agent line #   2: saw an allow or disallow line state = 0 linenumber = 0 entry = Entry ( ) self . modified ( ) for line in lines : linenumber += 1 if not line : if state == 1 : entry = Entry ( ) state = 0 elif state == 2 : self . _add_entry ( entry ) entry = Entry ( ) state = 0 # remove optional comment and strip line i = line . find ( '#' ) if i >= 0 : line = line [ : i ] line = line . strip ( ) if not line : continue line = line . split ( ':' , 1 ) if len ( line ) == 2 : line [ 0 ] = line [ 0 ] . strip ( ) . lower ( ) line [ 1 ] = urllib . unquote ( line [ 1 ] . strip ( ) ) if line [ 0 ] == "user-agent" : if state == 2 : self . _add_entry ( entry ) entry = Entry ( ) entry . useragents . append ( line [ 1 ] ) state = 1 elif line [ 0 ] == "disallow" : if state != 0 : entry . rulelines . append ( RuleLine ( line [ 1 ] , False ) ) state = 2 elif line [ 0 ] == "allow" : if state != 0 : entry . rulelines . append ( RuleLine ( line [ 1 ] , True ) ) state = 2 if state == 2 : self . _add_entry ( entry )