parameters: (readline)
docstring: wrap generate_tokens to catch EOF errors
docstring_summary: wrap generate_tokens to catch EOF errors
docstring_tokens: wrap generate_tokens to catch EOF errors
function: def generate_tokens(readline):
      """wrap generate_tokens to catch EOF errors"""
      try:
          for token in tokenize2.generate_tokens(readline):
              yield token
      except tokenize2.TokenError:
          # catch EOF error
          return
function_tokens: def generate_tokens ( readline ) : try : for token in tokenize2 . generate_tokens ( readline ) : yield token except tokenize2 . TokenError : # catch EOF error return