def tokenize(self, text):
    'Tokenizes a piece of text into its word pieces.\n\n        This uses a greedy longest-match-first algorithm to perform tokenization\n        using the given vocabulary.\n\n        For example:\n          input = "unaffable"\n          output = ["un", "##aff", "##able"]\n\n        Args:\n          text: A single token or whitespace separated tokens. This should have\n            already been passed through `BasicTokenizer.\n\n        Returns:\n          A list of wordpiece tokens.\n        '
    text = convert_to_unicode(text)
    output_tokens = []
    for token in whitespace_tokenize(text):
        chars = list(token)
        if (len(chars) > self.max_input_chars_per_word):
            output_tokens.append(self.unk_token)
            continue
        is_bad = False
        start = 0
        sub_tokens = []
        while (start < len(chars)):
            end = len(chars)
            cur_substr = None
            while (start < end):
                substr = ''.join(chars[start:end])
                if (start > 0):
                    substr = ('##' + substr)
                if (substr in self.vocab):
                    cur_substr = substr
                    break
                end -= 1
            if (cur_substr is None):
                is_bad = True
                break
            sub_tokens.append(cur_substr)
            start = end
        if is_bad:
            output_tokens.append(self.unk_token)
        else:
            output_tokens.extend(sub_tokens)
    return output_tokens
Tokenizes a piece of text into its word pieces.

This uses a greedy longest-match-first algorithm to perform tokenization
using the given vocabulary.

For example:
  input = "unaffable"
  output = ["un", "##aff", "##able"]

Args:
  text: A single token or whitespace separated tokens. This should have
    already been passed through `BasicTokenizer.

Returns:
  A list of wordpiece tokens.
def tokenize(self, text):
    'Tokenizes a piece of text into its word pieces.\n\n        This uses a greedy longest-match-first algorithm to perform tokenization\n        using the given vocabulary.\n\n        For example:\n          input = "unaffable"\n          output = ["un", "##aff", "##able"]\n\n        Args:\n          text: A single token or whitespace separated tokens. This should have\n            already been passed through `BasicTokenizer.\n\n        Returns:\n          A list of wordpiece tokens.\n        '
    text = convert_to_unicode(text)
    output_tokens = []
    for token in whitespace_tokenize(text):
        chars = list(token)
        if (len(chars) > self.max_input_chars_per_word):
            output_tokens.append(self.unk_token)
            continue
        is_bad = False
        start = 0
        sub_tokens = []
        while (start < len(chars)):
            end = len(chars)
            cur_substr = None
            while (start < end):
                substr = .join(chars[start:end])
                if (start > 0):
                    substr = ('##' + substr)
                if (substr in self.vocab):
                    cur_substr = substr
                    break
                end -= 1
            if (cur_substr is None):
                is_bad = True
                break
            sub_tokens.append(cur_substr)
            start = end
        if is_bad:
            output_tokens.append(self.unk_token)
        else:
            output_tokens.extend(sub_tokens)
    return output_tokens