parameters: (self)
return_statement: return token
docstring: This function breaks the time string into lexical units (tokens), which
          can be parsed by the parser. Lexical units are demarcated by changes in
          the character set, so any continuous string of letters is considered
          one unit, any continuous string of numbers is considered one unit.
  
          The main complication arises from the fact that dots ('.') can be used
          both as separators (e.g. "Sep.20.2009") or decimal points (e.g.
          "4:30:21.447"). As such, it is necessary to read the full context of
          any dot-separated strings before breaking it into tokens; as such, this
          function maintains a "token stack", for when the ambiguous context
          demands that multiple tokens be parsed at once.
docstring_summary: This function breaks the time string into lexical units (tokens), which
          can be parsed by the parser. Lexical units are demarcated by changes in
          the character set, so any continuous string of letters is considered
          one unit, any continuous string of numbers is considered one unit.
docstring_tokens: This function breaks the time string into lexical units ( tokens ) which can be parsed by the parser . Lexical units are demarcated by changes in the character set so any continuous string of letters is considered one unit any continuous string of numbers is considered one unit .
function: def get_token(self):
          """
          This function breaks the time string into lexical units (tokens), which
          can be parsed by the parser. Lexical units are demarcated by changes in
          the character set, so any continuous string of letters is considered
          one unit, any continuous string of numbers is considered one unit.
  
          The main complication arises from the fact that dots ('.') can be used
          both as separators (e.g. "Sep.20.2009") or decimal points (e.g.
          "4:30:21.447"). As such, it is necessary to read the full context of
          any dot-separated strings before breaking it into tokens; as such, this
          function maintains a "token stack", for when the ambiguous context
          demands that multiple tokens be parsed at once.
          """
          if self.tokenstack:
              return self.tokenstack.pop(0)
  
          seenletters = False
          token = None
          state = None
  
          while not self.eof:
              # We only realize that we've reached the end of a token when we
              # find a character that's not part of the current token - since
              # that character may be part of the next token, it's stored in the
              # charstack.
              if self.charstack:
                  nextchar = self.charstack.pop(0)
              else:
                  nextchar = self.instream.read(1)
                  while nextchar == '\x00':
                      nextchar = self.instream.read(1)
  
              if not nextchar:
                  self.eof = True
                  break
              elif not state:
                  # First character of the token - determines if we're starting
                  # to parse a word, a number or something else.
                  token = nextchar
                  if self.isword(nextchar):
                      state = 'a'
                  elif self.isnum(nextchar):
                      state = '0'
                  elif self.isspace(nextchar):
                      token = ' '
                      break  # emit token
                  else:
                      break  # emit token
              elif state == 'a':
                  # If we've already started reading a word, we keep reading
                  # letters until we find something that's not part of a word.
                  seenletters = True
                  if self.isword(nextchar):
                      token += nextchar
                  elif nextchar == '.':
                      token += nextchar
                      state = 'a.'
                  else:
                      self.charstack.append(nextchar)
                      break  # emit token
              elif state == '0':
                  # If we've already started reading a number, we keep reading
                  # numbers until we find something that doesn't fit.
                  if self.isnum(nextchar):
                      token += nextchar
                  elif nextchar == '.' or (nextchar == ',' and len(token) >= 2):
                      token += nextchar
                      state = '0.'
                  else:
                      self.charstack.append(nextchar)
                      break  # emit token
              elif state == 'a.':
                  # If we've seen some letters and a dot separator, continue
                  # parsing, and the tokens will be broken up later.
                  seenletters = True
                  if nextchar == '.' or self.isword(nextchar):
                      token += nextchar
                  elif self.isnum(nextchar) and token[-1] == '.':
                      token += nextchar
                      state = '0.'
                  else:
                      self.charstack.append(nextchar)
                      break  # emit token
              elif state == '0.':
                  # If we've seen at least one dot separator, keep going, we'll
                  # break up the tokens later.
                  if nextchar == '.' or self.isnum(nextchar):
                      token += nextchar
                  elif self.isword(nextchar) and token[-1] == '.':
                      token += nextchar
                      state = 'a.'
                  else:
                      self.charstack.append(nextchar)
                      break  # emit token
  
          if (state in ('a.', '0.') and (seenletters or token.count('.') > 1 or
                                         token[-1] in '.,')):
              l = self._split_decimal.split(token)
              token = l[0]
              for tok in l[1:]:
                  if tok:
                      self.tokenstack.append(tok)
  
          if state == '0.' and token.count('.') == 0:
              token = token.replace(',', '.')
  
          return token
function_tokens: def get_token ( self ) : if self . tokenstack : return self . tokenstack . pop ( 0 ) seenletters = False token = None state = None while not self . eof : # We only realize that we've reached the end of a token when we # find a character that's not part of the current token - since # that character may be part of the next token, it's stored in the # charstack. if self . charstack : nextchar = self . charstack . pop ( 0 ) else : nextchar = self . instream . read ( 1 ) while nextchar == '\x00' : nextchar = self . instream . read ( 1 ) if not nextchar : self . eof = True break elif not state : # First character of the token - determines if we're starting # to parse a word, a number or something else. token = nextchar if self . isword ( nextchar ) : state = 'a' elif self . isnum ( nextchar ) : state = '0' elif self . isspace ( nextchar ) : token = ' ' break # emit token else : break # emit token elif state == 'a' : # If we've already started reading a word, we keep reading # letters until we find something that's not part of a word. seenletters = True if self . isword ( nextchar ) : token += nextchar elif nextchar == '.' : token += nextchar state = 'a.' else : self . charstack . append ( nextchar ) break # emit token elif state == '0' : # If we've already started reading a number, we keep reading # numbers until we find something that doesn't fit. if self . isnum ( nextchar ) : token += nextchar elif nextchar == '.' or ( nextchar == ',' and len ( token ) >= 2 ) : token += nextchar state = '0.' else : self . charstack . append ( nextchar ) break # emit token elif state == 'a.' : # If we've seen some letters and a dot separator, continue # parsing, and the tokens will be broken up later. seenletters = True if nextchar == '.' or self . isword ( nextchar ) : token += nextchar elif self . isnum ( nextchar ) and token [ - 1 ] == '.' : token += nextchar state = '0.' else : self . charstack . append ( nextchar ) break # emit token elif state == '0.' : # If we've seen at least one dot separator, keep going, we'll # break up the tokens later. if nextchar == '.' or self . isnum ( nextchar ) : token += nextchar elif self . isword ( nextchar ) and token [ - 1 ] == '.' : token += nextchar state = 'a.' else : self . charstack . append ( nextchar ) break # emit token if ( state in ( 'a.' , '0.' ) and ( seenletters or token . count ( '.' ) > 1 or token [ - 1 ] in '.,' ) ) : l = self . _split_decimal . split ( token ) token = l [ 0 ] for tok in l [ 1 : ] : if tok : self . tokenstack . append ( tok ) if state == '0.' and token . count ( '.' ) == 0 : token = token . replace ( ',' , '.' ) return token