parameters: (readline)
docstring: The generate_tokens() generator requires one argment, readline, which
      must be a callable object which provides the same interface as the
      readline() method of built-in file objects. Each call to the function
      should return one line of input as a string.  Alternately, readline
      can be a callable function terminating with StopIteration:
          readline = open(myfile).next    # Example of alternate readline
  
      The generator produces 5-tuples with these members: the token type; the
      token string; a 2-tuple (srow, scol) of ints specifying the row and
      column where the token begins in the source; a 2-tuple (erow, ecol) of
      ints specifying the row and column where the token ends in the source;
      and the line on which the token was found. The line passed is the
      logical line; continuation lines are included.
docstring_summary: The generate_tokens() generator requires one argment, readline, which
      must be a callable object which provides the same interface as the
      readline() method of built-in file objects. Each call to the function
      should return one line of input as a string.  Alternately, readline
      can be a callable function terminating with StopIteration:
          readline = open(myfile).next    # Example of alternate readline
docstring_tokens: The generate_tokens () generator requires one argment readline which must be a callable object which provides the same interface as the readline () method of built - in file objects . Each call to the function should return one line of input as a string . Alternately readline can be a callable function terminating with StopIteration : readline = open ( myfile ) . next # Example of alternate readline
function: def generate_tokens(readline):
      """
      The generate_tokens() generator requires one argment, readline, which
      must be a callable object which provides the same interface as the
      readline() method of built-in file objects. Each call to the function
      should return one line of input as a string.  Alternately, readline
      can be a callable function terminating with StopIteration:
          readline = open(myfile).next    # Example of alternate readline
  
      The generator produces 5-tuples with these members: the token type; the
      token string; a 2-tuple (srow, scol) of ints specifying the row and
      column where the token begins in the source; a 2-tuple (erow, ecol) of
      ints specifying the row and column where the token ends in the source;
      and the line on which the token was found. The line passed is the
      logical line; continuation lines are included.
      """
      lnum = parenlev = continued = 0
      namechars, numchars = string.ascii_letters + '_', '0123456789'
      contstr, needcont = '', 0
      contline = None
      indents = [0]
  
      while 1:                                   # loop over lines in stream
          try:
              line = readline()
          except StopIteration:
              line = ''
          lnum = lnum + 1
          pos, max = 0, len(line)
  
          if contstr:                            # continued string
              if not line:
                  raise TokenError, ("EOF in multi-line string", strstart)
              endmatch = endprog.match(line)
              if endmatch:
                  pos = end = endmatch.end(0)
                  yield (STRING, contstr + line[:end],
                         strstart, (lnum, end), contline + line)
                  contstr, needcont = '', 0
                  contline = None
              elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
                  yield (ERRORTOKEN, contstr + line,
                             strstart, (lnum, len(line)), contline)
                  contstr = ''
                  contline = None
                  continue
              else:
                  contstr = contstr + line
                  contline = contline + line
                  continue
  
          elif parenlev == 0 and not continued:  # new statement
              if not line: break
              column = 0
              while pos < max:                   # measure leading whitespace
                  if line[pos] == ' ': column = column + 1
                  elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
                  elif line[pos] == '\f': column = 0
                  else: break
                  pos = pos + 1
              if pos == max: break
  
              if line[pos] in '#\r\n':           # skip comments or blank lines
                  if line[pos] == '#':
                      comment_token = line[pos:].rstrip('\r\n')
                      nl_pos = pos + len(comment_token)
                      yield (COMMENT, comment_token,
                             (lnum, pos), (lnum, pos + len(comment_token)), line)
                      yield (NL, line[nl_pos:],
                             (lnum, nl_pos), (lnum, len(line)), line)
                  else:
                      yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
                             (lnum, pos), (lnum, len(line)), line)
                  continue
  
              if column > indents[-1]:           # count indents or dedents
                  indents.append(column)
                  yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
              while column < indents[-1]:
                  if column not in indents:
                      raise IndentationError(
                          "unindent does not match any outer indentation level",
                          ("<tokenize>", lnum, pos, line))
                  indents = indents[:-1]
                  yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
  
          else:                                  # continued statement
              if not line:
                  raise TokenError, ("EOF in multi-line statement", (lnum, 0))
              continued = 0
  
          while pos < max:
              pseudomatch = pseudoprog.match(line, pos)
              if pseudomatch:                                # scan for tokens
                  start, end = pseudomatch.span(1)
                  spos, epos, pos = (lnum, start), (lnum, end), end
                  token, initial = line[start:end], line[start]
  
                  if initial in numchars or \
                     (initial == '.' and token != '.'):      # ordinary number
                      yield (NUMBER, token, spos, epos, line)
                  elif initial in '\r\n':
                      yield (NL if parenlev > 0 else NEWLINE,
                             token, spos, epos, line)
                  elif initial == '#':
                      assert not token.endswith("\n")
                      yield (COMMENT, token, spos, epos, line)
                  elif token in triple_quoted:
                      endprog = endprogs[token]
                      endmatch = endprog.match(line, pos)
                      if endmatch:                           # all on one line
                          pos = endmatch.end(0)
                          token = line[start:pos]
                          yield (STRING, token, spos, (lnum, pos), line)
                      else:
                          strstart = (lnum, start)           # multiple lines
                          contstr = line[start:]
                          contline = line
                          break
                  elif initial in single_quoted or \
                      token[:2] in single_quoted or \
                      token[:3] in single_quoted:
                      if token[-1] == '\n':                  # continued string
                          strstart = (lnum, start)
                          endprog = (endprogs[initial] or endprogs[token[1]] or
                                     endprogs[token[2]])
                          contstr, needcont = line[start:], 1
                          contline = line
                          break
                      else:                                  # ordinary string
                          yield (STRING, token, spos, epos, line)
                  elif initial in namechars:                 # ordinary name
                      yield (NAME, token, spos, epos, line)
                  elif initial == '\\':                      # continued stmt
                      continued = 1
                  else:
                      if initial in '([{': parenlev = parenlev + 1
                      elif initial in ')]}': parenlev = parenlev - 1
                      yield (OP, token, spos, epos, line)
              else:
                  yield (ERRORTOKEN, line[pos],
                             (lnum, pos), (lnum, pos+1), line)
                  pos = pos + 1
  
      for indent in indents[1:]:                 # pop remaining indent levels
          yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
      yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
function_tokens: def generate_tokens ( readline ) : lnum = parenlev = continued = 0 namechars , numchars = string . ascii_letters + '_' , '0123456789' contstr , needcont = '' , 0 contline = None indents = [ 0 ] while 1 : # loop over lines in stream try : line = readline ( ) except StopIteration : line = '' lnum = lnum + 1 pos , max = 0 , len ( line ) if contstr : # continued string if not line : raise TokenError , ( "EOF in multi-line string" , strstart ) endmatch = endprog . match ( line ) if endmatch : pos = end = endmatch . end ( 0 ) yield ( STRING , contstr + line [ : end ] , strstart , ( lnum , end ) , contline + line ) contstr , needcont = '' , 0 contline = None elif needcont and line [ - 2 : ] != '\\\n' and line [ - 3 : ] != '\\\r\n' : yield ( ERRORTOKEN , contstr + line , strstart , ( lnum , len ( line ) ) , contline ) contstr = '' contline = None continue else : contstr = contstr + line contline = contline + line continue elif parenlev == 0 and not continued : # new statement if not line : break column = 0 while pos < max : # measure leading whitespace if line [ pos ] == ' ' : column = column + 1 elif line [ pos ] == '\t' : column = ( column / tabsize + 1 ) * tabsize elif line [ pos ] == '\f' : column = 0 else : break pos = pos + 1 if pos == max : break if line [ pos ] in '#\r\n' : # skip comments or blank lines if line [ pos ] == '#' : comment_token = line [ pos : ] . rstrip ( '\r\n' ) nl_pos = pos + len ( comment_token ) yield ( COMMENT , comment_token , ( lnum , pos ) , ( lnum , pos + len ( comment_token ) ) , line ) yield ( NL , line [ nl_pos : ] , ( lnum , nl_pos ) , ( lnum , len ( line ) ) , line ) else : yield ( ( NL , COMMENT ) [ line [ pos ] == '#' ] , line [ pos : ] , ( lnum , pos ) , ( lnum , len ( line ) ) , line ) continue if column > indents [ - 1 ] : # count indents or dedents indents . append ( column ) yield ( INDENT , line [ : pos ] , ( lnum , 0 ) , ( lnum , pos ) , line ) while column < indents [ - 1 ] : if column not in indents : raise IndentationError ( "unindent does not match any outer indentation level" , ( "<tokenize>" , lnum , pos , line ) ) indents = indents [ : - 1 ] yield ( DEDENT , '' , ( lnum , pos ) , ( lnum , pos ) , line ) else : # continued statement if not line : raise TokenError , ( "EOF in multi-line statement" , ( lnum , 0 ) ) continued = 0 while pos < max : pseudomatch = pseudoprog . match ( line , pos ) if pseudomatch : # scan for tokens start , end = pseudomatch . span ( 1 ) spos , epos , pos = ( lnum , start ) , ( lnum , end ) , end token , initial = line [ start : end ] , line [ start ] if initial in numchars or ( initial == '.' and token != '.' ) : # ordinary number yield ( NUMBER , token , spos , epos , line ) elif initial in '\r\n' : yield ( NL if parenlev > 0 else NEWLINE , token , spos , epos , line ) elif initial == '#' : assert not token . endswith ( "\n" ) yield ( COMMENT , token , spos , epos , line ) elif token in triple_quoted : endprog = endprogs [ token ] endmatch = endprog . match ( line , pos ) if endmatch : # all on one line pos = endmatch . end ( 0 ) token = line [ start : pos ] yield ( STRING , token , spos , ( lnum , pos ) , line ) else : strstart = ( lnum , start ) # multiple lines contstr = line [ start : ] contline = line break elif initial in single_quoted or token [ : 2 ] in single_quoted or token [ : 3 ] in single_quoted : if token [ - 1 ] == '\n' : # continued string strstart = ( lnum , start ) endprog = ( endprogs [ initial ] or endprogs [ token [ 1 ] ] or endprogs [ token [ 2 ] ] ) contstr , needcont = line [ start : ] , 1 contline = line break else : # ordinary string yield ( STRING , token , spos , epos , line ) elif initial in namechars : # ordinary name yield ( NAME , token , spos , epos , line ) elif initial == '\\' : # continued stmt continued = 1 else : if initial in '([{' : parenlev = parenlev + 1 elif initial in ')]}' : parenlev = parenlev - 1 yield ( OP , token , spos , epos , line ) else : yield ( ERRORTOKEN , line [ pos ] , ( lnum , pos ) , ( lnum , pos + 1 ) , line ) pos = pos + 1 for indent in indents [ 1 : ] : # pop remaining indent levels yield ( DEDENT , '' , ( lnum , 0 ) , ( lnum , 0 ) , '' ) yield ( ENDMARKER , '' , ( lnum , 0 ) , ( lnum , 0 ) , '' )