parameters: (self, text)
docstring: Parse JSON data.
docstring_summary: Parse JSON data.
docstring_tokens: Parse JSON data .
function: def get_tokens_unprocessed(self, text):
          """Parse JSON data."""
  
          in_string = False
          in_escape = False
          in_unicode_escape = 0
          in_whitespace = False
          in_constant = False
          in_number = False
          in_float = False
          in_punctuation = False
  
          start = 0
  
          # The queue is used to store data that may need to be tokenized
          # differently based on what follows. In particular, JSON object
          # keys are tokenized differently than string values, but cannot
          # be distinguished until punctuation is encountered outside the
          # string.
          #
          # A ":" character after the string indicates that the string is
          # an object key; any other character indicates the string is a
          # regular string value.
          #
          # The queue holds tuples that contain the following data:
          #
          #     (start_index, token_type, text)
          #
          # By default the token type of text in double quotes is
          # String.Double. The token type will be replaced if a colon
          # is encountered after the string closes.
          #
          queue = []
  
          for stop, character in enumerate(text):
              if in_string:
                  if in_unicode_escape:
                      if character in self.hexadecimals:
                          in_unicode_escape -= 1
                          if not in_unicode_escape:
                              in_escape = False
                      else:
                          in_unicode_escape = 0
                          in_escape = False
  
                  elif in_escape:
                      if character == 'u':
                          in_unicode_escape = 4
                      else:
                          in_escape = False
  
                  elif character == '\\':
                      in_escape = True
  
                  elif character == '"':
                      queue.append((start, String.Double, text[start:stop + 1]))
                      in_string = False
                      in_escape = False
                      in_unicode_escape = 0
  
                  continue
  
              elif in_whitespace:
                  if character in self.whitespaces:
                      continue
  
                  if queue:
                      queue.append((start, Whitespace, text[start:stop]))
                  else:
                      yield start, Whitespace, text[start:stop]
                  in_whitespace = False
                  # Fall through so the new character can be evaluated.
  
              elif in_constant:
                  if character in self.constants:
                      continue
  
                  yield start, Keyword.Constant, text[start:stop]
                  in_constant = False
                  # Fall through so the new character can be evaluated.
  
              elif in_number:
                  if character in self.integers:
                      continue
                  elif character in self.floats:
                      in_float = True
                      continue
  
                  if in_float:
                      yield start, Number.Float, text[start:stop]
                  else:
                      yield start, Number.Integer, text[start:stop]
                  in_number = False
                  in_float = False
                  # Fall through so the new character can be evaluated.
  
              elif in_punctuation:
                  if character in self.punctuations:
                      continue
  
                  yield start, Punctuation, text[start:stop]
                  in_punctuation = False
                  # Fall through so the new character can be evaluated.
  
              start = stop
  
              if character == '"':
                  in_string = True
  
              elif character in self.whitespaces:
                  in_whitespace = True
  
              elif character in {'f', 'n', 't'}:  # The first letters of true|false|null
                  # Exhaust the queue. Accept the existing token types.
                  yield from queue
                  queue.clear()
  
                  in_constant = True
  
              elif character in self.integers:
                  # Exhaust the queue. Accept the existing token types.
                  yield from queue
                  queue.clear()
  
                  in_number = True
  
              elif character == ':':
                  # Yield from the queue. Replace string token types.
                  for _start, _token, _text in queue:
                      # There can be only two types of tokens before a ':':
                      # Whitespace, or a quoted string. If it's a quoted string
                      # we emit Name.Tag, otherwise, we yield the whitespace
                      # tokens. In all other cases this is invalid JSON. This
                      # allows for things like '"foo" "bar": "baz"' but we're not
                      # a validating JSON lexer so it's acceptable
                      if _token is Whitespace:
                          yield _start, _token, _text
                      elif _token is String.Double:
                          yield _start, Name.Tag, _text
                      else:
                          yield _start, Error, _text
                  queue.clear()
  
                  in_punctuation = True
  
              elif character in self.punctuations:
                  # Exhaust the queue. Accept the existing token types.
                  yield from queue
                  queue.clear()
  
                  in_punctuation = True
  
              else:
                  # Exhaust the queue. Accept the existing token types.
                  yield from queue
                  queue.clear()
  
                  yield start, Error, character
  
          # Yield any remaining text.
          yield from queue
          if in_string:
              yield start, Error, text[start:]
          elif in_float:
              yield start, Number.Float, text[start:]
          elif in_number:
              yield start, Number.Integer, text[start:]
          elif in_constant:
              yield start, Keyword.Constant, text[start:]
          elif in_whitespace:
              yield start, Whitespace, text[start:]
          elif in_punctuation:
              yield start, Punctuation, text[start:]
function_tokens: def get_tokens_unprocessed ( self , text ) : in_string = False in_escape = False in_unicode_escape = 0 in_whitespace = False in_constant = False in_number = False in_float = False in_punctuation = False start = 0 # The queue is used to store data that may need to be tokenized # differently based on what follows. In particular, JSON object # keys are tokenized differently than string values, but cannot # be distinguished until punctuation is encountered outside the # string. # # A ":" character after the string indicates that the string is # an object key; any other character indicates the string is a # regular string value. # # The queue holds tuples that contain the following data: # #     (start_index, token_type, text) # # By default the token type of text in double quotes is # String.Double. The token type will be replaced if a colon # is encountered after the string closes. # queue = [ ] for stop , character in enumerate ( text ) : if in_string : if in_unicode_escape : if character in self . hexadecimals : in_unicode_escape -= 1 if not in_unicode_escape : in_escape = False else : in_unicode_escape = 0 in_escape = False elif in_escape : if character == 'u' : in_unicode_escape = 4 else : in_escape = False elif character == '\\' : in_escape = True elif character == '"' : queue . append ( ( start , String . Double , text [ start : stop + 1 ] ) ) in_string = False in_escape = False in_unicode_escape = 0 continue elif in_whitespace : if character in self . whitespaces : continue if queue : queue . append ( ( start , Whitespace , text [ start : stop ] ) ) else : yield start , Whitespace , text [ start : stop ] in_whitespace = False # Fall through so the new character can be evaluated. elif in_constant : if character in self . constants : continue yield start , Keyword . Constant , text [ start : stop ] in_constant = False # Fall through so the new character can be evaluated. elif in_number : if character in self . integers : continue elif character in self . floats : in_float = True continue if in_float : yield start , Number . Float , text [ start : stop ] else : yield start , Number . Integer , text [ start : stop ] in_number = False in_float = False # Fall through so the new character can be evaluated. elif in_punctuation : if character in self . punctuations : continue yield start , Punctuation , text [ start : stop ] in_punctuation = False # Fall through so the new character can be evaluated. start = stop if character == '"' : in_string = True elif character in self . whitespaces : in_whitespace = True elif character in { 'f' , 'n' , 't' } : # The first letters of true|false|null # Exhaust the queue. Accept the existing token types. yield from queue queue . clear ( ) in_constant = True elif character in self . integers : # Exhaust the queue. Accept the existing token types. yield from queue queue . clear ( ) in_number = True elif character == ':' : # Yield from the queue. Replace string token types. for _start , _token , _text in queue : # There can be only two types of tokens before a ':': # Whitespace, or a quoted string. If it's a quoted string # we emit Name.Tag, otherwise, we yield the whitespace # tokens. In all other cases this is invalid JSON. This # allows for things like '"foo" "bar": "baz"' but we're not # a validating JSON lexer so it's acceptable if _token is Whitespace : yield _start , _token , _text elif _token is String . Double : yield _start , Name . Tag , _text else : yield _start , Error , _text queue . clear ( ) in_punctuation = True elif character in self . punctuations : # Exhaust the queue. Accept the existing token types. yield from queue queue . clear ( ) in_punctuation = True else : # Exhaust the queue. Accept the existing token types. yield from queue queue . clear ( ) yield start , Error , character # Yield any remaining text. yield from queue if in_string : yield start , Error , text [ start : ] elif in_float : yield start , Number . Float , text [ start : ] elif in_number : yield start , Number . Integer , text [ start : ] elif in_constant : yield start , Keyword . Constant , text [ start : ] elif in_whitespace : yield start , Whitespace , text [ start : ] elif in_punctuation : yield start , Punctuation , text [ start : ]