parameters: (
          self,
          source: str,
          name: t.Optional[str],
          filename: t.Optional[str] = None,
          state: t.Optional[str] = None,
      )
docstring: This method tokenizes the text and returns the tokens in a
          generator. Use this method if you just want to tokenize a template.
  
          .. versionchanged:: 3.0
              Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
              breaks.
docstring_summary: This method tokenizes the text and returns the tokens in a
          generator. Use this method if you just want to tokenize a template.
docstring_tokens: This method tokenizes the text and returns the tokens in a generator . Use this method if you just want to tokenize a template .
function: def tokeniter(
          self,
          source: str,
          name: t.Optional[str],
          filename: t.Optional[str] = None,
          state: t.Optional[str] = None,
      ) -> t.Iterator[t.Tuple[int, str, str]]:
          """This method tokenizes the text and returns the tokens in a
          generator. Use this method if you just want to tokenize a template.
  
          .. versionchanged:: 3.0
              Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
              breaks.
          """
          lines = newline_re.split(source)[::2]
  
          if not self.keep_trailing_newline and lines[-1] == "":
              del lines[-1]
  
          source = "\n".join(lines)
          pos = 0
          lineno = 1
          stack = ["root"]
  
          if state is not None and state != "root":
              assert state in ("variable", "block"), "invalid state"
              stack.append(state + "_begin")
  
          statetokens = self.rules[stack[-1]]
          source_length = len(source)
          balancing_stack: t.List[str] = []
          lstrip_unless_re = self.lstrip_unless_re
          newlines_stripped = 0
          line_starting = True
  
          while True:
              # tokenizer loop
              for regex, tokens, new_state in statetokens:
                  m = regex.match(source, pos)
  
                  # if no match we try again with the next rule
                  if m is None:
                      continue
  
                  # we only match blocks and variables if braces / parentheses
                  # are balanced. continue parsing with the lower rule which
                  # is the operator rule. do this only if the end tags look
                  # like operators
                  if balancing_stack and tokens in (
                      TOKEN_VARIABLE_END,
                      TOKEN_BLOCK_END,
                      TOKEN_LINESTATEMENT_END,
                  ):
                      continue
  
                  # tuples support more options
                  if isinstance(tokens, tuple):
                      groups = m.groups()
  
                      if isinstance(tokens, OptionalLStrip):
                          # Rule supports lstrip. Match will look like
                          # text, block type, whitespace control, type, control, ...
                          text = groups[0]
                          # Skipping the text and first type, every other group is the
                          # whitespace control for each type. One of the groups will be
                          # -, +, or empty string instead of None.
                          strip_sign = next(g for g in groups[2::2] if g is not None)
  
                          if strip_sign == "-":
                              # Strip all whitespace between the text and the tag.
                              stripped = text.rstrip()
                              newlines_stripped = text[len(stripped) :].count("\n")
                              groups = [stripped, *groups[1:]]
                          elif (
                              # Not marked for preserving whitespace.
                              strip_sign != "+"
                              # lstrip is enabled.
                              and lstrip_unless_re is not None
                              # Not a variable expression.
                              and not m.groupdict().get(TOKEN_VARIABLE_BEGIN)
                          ):
                              # The start of text between the last newline and the tag.
                              l_pos = text.rfind("\n") + 1
  
                              if l_pos > 0 or line_starting:
                                  # If there's only whitespace between the newline and the
                                  # tag, strip it.
                                  if not lstrip_unless_re.search(text, l_pos):
                                      groups = [text[:l_pos], *groups[1:]]
  
                      for idx, token in enumerate(tokens):
                          # failure group
                          if token.__class__ is Failure:
                              raise token(lineno, filename)
                          # bygroup is a bit more complex, in that case we
                          # yield for the current token the first named
                          # group that matched
                          elif token == "#bygroup":
                              for key, value in m.groupdict().items():
                                  if value is not None:
                                      yield lineno, key, value
                                      lineno += value.count("\n")
                                      break
                              else:
                                  raise RuntimeError(
                                      f"{regex!r} wanted to resolve the token dynamically"
                                      " but no group matched"
                                  )
                          # normal group
                          else:
                              data = groups[idx]
  
                              if data or token not in ignore_if_empty:
                                  yield lineno, token, data
  
                              lineno += data.count("\n") + newlines_stripped
                              newlines_stripped = 0
  
                  # strings as token just are yielded as it.
                  else:
                      data = m.group()
  
                      # update brace/parentheses balance
                      if tokens == TOKEN_OPERATOR:
                          if data == "{":
                              balancing_stack.append("}")
                          elif data == "(":
                              balancing_stack.append(")")
                          elif data == "[":
                              balancing_stack.append("]")
                          elif data in ("}", ")", "]"):
                              if not balancing_stack:
                                  raise TemplateSyntaxError(
                                      f"unexpected '{data}'", lineno, name, filename
                                  )
  
                              expected_op = balancing_stack.pop()
  
                              if expected_op != data:
                                  raise TemplateSyntaxError(
                                      f"unexpected '{data}', expected '{expected_op}'",
                                      lineno,
                                      name,
                                      filename,
                                  )
  
                      # yield items
                      if data or tokens not in ignore_if_empty:
                          yield lineno, tokens, data
  
                      lineno += data.count("\n")
  
                  line_starting = m.group()[-1:] == "\n"
                  # fetch new position into new variable so that we can check
                  # if there is a internal parsing error which would result
                  # in an infinite loop
                  pos2 = m.end()
  
                  # handle state changes
                  if new_state is not None:
                      # remove the uppermost state
                      if new_state == "#pop":
                          stack.pop()
                      # resolve the new state by group checking
                      elif new_state == "#bygroup":
                          for key, value in m.groupdict().items():
                              if value is not None:
                                  stack.append(key)
                                  break
                          else:
                              raise RuntimeError(
                                  f"{regex!r} wanted to resolve the new state dynamically"
                                  f" but no group matched"
                              )
                      # direct state name given
                      else:
                          stack.append(new_state)
  
                      statetokens = self.rules[stack[-1]]
                  # we are still at the same position and no stack change.
                  # this means a loop without break condition, avoid that and
                  # raise error
                  elif pos2 == pos:
                      raise RuntimeError(
                          f"{regex!r} yielded empty string without stack change"
                      )
  
                  # publish new function and start again
                  pos = pos2
                  break
              # if loop terminated without break we haven't found a single match
              # either we are at the end of the file or we have a problem
              else:
                  # end of text
                  if pos >= source_length:
                      return
  
                  # something went wrong
                  raise TemplateSyntaxError(
                      f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename
                  )
function_tokens: def tokeniter ( self , source : str , name : t . Optional [ str ] , filename : t . Optional [ str ] = None , state : t . Optional [ str ] = None , ) -> t . Iterator [ t . Tuple [ int , str , str ] ] : lines = newline_re . split ( source ) [ : : 2 ] if not self . keep_trailing_newline and lines [ - 1 ] == "" : del lines [ - 1 ] source = "\n" . join ( lines ) pos = 0 lineno = 1 stack = [ "root" ] if state is not None and state != "root" : assert state in ( "variable" , "block" ) , "invalid state" stack . append ( state + "_begin" ) statetokens = self . rules [ stack [ - 1 ] ] source_length = len ( source ) balancing_stack : t . List [ str ] = [ ] lstrip_unless_re = self . lstrip_unless_re newlines_stripped = 0 line_starting = True while True : # tokenizer loop for regex , tokens , new_state in statetokens : m = regex . match ( source , pos ) # if no match we try again with the next rule if m is None : continue # we only match blocks and variables if braces / parentheses # are balanced. continue parsing with the lower rule which # is the operator rule. do this only if the end tags look # like operators if balancing_stack and tokens in ( TOKEN_VARIABLE_END , TOKEN_BLOCK_END , TOKEN_LINESTATEMENT_END , ) : continue # tuples support more options if isinstance ( tokens , tuple ) : groups = m . groups ( ) if isinstance ( tokens , OptionalLStrip ) : # Rule supports lstrip. Match will look like # text, block type, whitespace control, type, control, ... text = groups [ 0 ] # Skipping the text and first type, every other group is the # whitespace control for each type. One of the groups will be # -, +, or empty string instead of None. strip_sign = next ( g for g in groups [ 2 : : 2 ] if g is not None ) if strip_sign == "-" : # Strip all whitespace between the text and the tag. stripped = text . rstrip ( ) newlines_stripped = text [ len ( stripped ) : ] . count ( "\n" ) groups = [ stripped , * groups [ 1 : ] ] elif ( # Not marked for preserving whitespace. strip_sign != "+" # lstrip is enabled. and lstrip_unless_re is not None # Not a variable expression. and not m . groupdict ( ) . get ( TOKEN_VARIABLE_BEGIN ) ) : # The start of text between the last newline and the tag. l_pos = text . rfind ( "\n" ) + 1 if l_pos > 0 or line_starting : # If there's only whitespace between the newline and the # tag, strip it. if not lstrip_unless_re . search ( text , l_pos ) : groups = [ text [ : l_pos ] , * groups [ 1 : ] ] for idx , token in enumerate ( tokens ) : # failure group if token . __class__ is Failure : raise token ( lineno , filename ) # bygroup is a bit more complex, in that case we # yield for the current token the first named # group that matched elif token == "#bygroup" : for key , value in m . groupdict ( ) . items ( ) : if value is not None : yield lineno , key , value lineno += value . count ( "\n" ) break else : raise RuntimeError ( f"{regex!r} wanted to resolve the token dynamically" " but no group matched" ) # normal group else : data = groups [ idx ] if data or token not in ignore_if_empty : yield lineno , token , data lineno += data . count ( "\n" ) + newlines_stripped newlines_stripped = 0 # strings as token just are yielded as it. else : data = m . group ( ) # update brace/parentheses balance if tokens == TOKEN_OPERATOR : if data == "{" : balancing_stack . append ( "}" ) elif data == "(" : balancing_stack . append ( ")" ) elif data == "[" : balancing_stack . append ( "]" ) elif data in ( "}" , ")" , "]" ) : if not balancing_stack : raise TemplateSyntaxError ( f"unexpected '{data}'" , lineno , name , filename ) expected_op = balancing_stack . pop ( ) if expected_op != data : raise TemplateSyntaxError ( f"unexpected '{data}', expected '{expected_op}'" , lineno , name , filename , ) # yield items if data or tokens not in ignore_if_empty : yield lineno , tokens , data lineno += data . count ( "\n" ) line_starting = m . group ( ) [ - 1 : ] == "\n" # fetch new position into new variable so that we can check # if there is a internal parsing error which would result # in an infinite loop pos2 = m . end ( ) # handle state changes if new_state is not None : # remove the uppermost state if new_state == "#pop" : stack . pop ( ) # resolve the new state by group checking elif new_state == "#bygroup" : for key , value in m . groupdict ( ) . items ( ) : if value is not None : stack . append ( key ) break else : raise RuntimeError ( f"{regex!r} wanted to resolve the new state dynamically" f" but no group matched" ) # direct state name given else : stack . append ( new_state ) statetokens = self . rules [ stack [ - 1 ] ] # we are still at the same position and no stack change. # this means a loop without break condition, avoid that and # raise error elif pos2 == pos : raise RuntimeError ( f"{regex!r} yielded empty string without stack change" ) # publish new function and start again pos = pos2 break # if loop terminated without break we haven't found a single match # either we are at the end of the file or we have a problem else : # end of text if pos >= source_length : return # something went wrong raise TemplateSyntaxError ( f"unexpected char {source[pos]!r} at {pos}" , lineno , name , filename )