parameters: (self, text)
return_statement: return output_tokens
docstring: Tokenizes a piece of text.
docstring_summary: Tokenizes a piece of text.
docstring_tokens: Tokenizes a piece of text .
function: def tokenize(self, text):
      """Tokenizes a piece of text."""
      text = convert_to_unicode(text)
      text = self._clean_text(text)
  
      # This was added on November 1st, 2018 for the multilingual and Chinese
      # models. This is also applied to the English models now, but it doesn't
      # matter since the English models were not trained on any Chinese data
      # and generally don't have any Chinese data in them (there are Chinese
      # characters in the vocabulary because Wikipedia does have some Chinese
      # words in the English Wikipedia.).
      text = self._tokenize_chinese_chars(text)
  
      orig_tokens = whitespace_tokenize(text)
      split_tokens = []
      for token in orig_tokens:
        if self.do_lower_case:
          token = token.lower()
          token = self._run_strip_accents(token)
        split_tokens.extend(self._run_split_on_punc(token))
  
      output_tokens = whitespace_tokenize(" ".join(split_tokens))
      return output_tokens
function_tokens: def tokenize ( self , text ) : text = convert_to_unicode ( text ) text = self . _clean_text ( text ) # This was added on November 1st, 2018 for the multilingual and Chinese # models. This is also applied to the English models now, but it doesn't # matter since the English models were not trained on any Chinese data # and generally don't have any Chinese data in them (there are Chinese # characters in the vocabulary because Wikipedia does have some Chinese # words in the English Wikipedia.). text = self . _tokenize_chinese_chars ( text ) orig_tokens = whitespace_tokenize ( text ) split_tokens = [ ] for token in orig_tokens : if self . do_lower_case : token = token . lower ( ) token = self . _run_strip_accents ( token ) split_tokens . extend ( self . _run_split_on_punc ( token ) ) output_tokens = whitespace_tokenize ( " " . join ( split_tokens ) ) return output_tokens