parameters: (sentence, vocabulary,
                            tokenizer=None, normalize_digits=True)
return_statement: return [vocabulary.get(_DIGIT_RE.sub(b"0", w), UNK_ID) for w in words]
docstring: Convert a string to list of integers representing token-ids.
  
    For example, a sentence "I have a dog" may become tokenized into
    ["I", "have", "a", "dog"] and with vocabulary {"I": 1, "have": 2,
    "a": 4, "dog": 7"} this function will return [1, 2, 4, 7].
  
    Args:
      sentence: the sentence in bytes format to convert to token-ids.
      vocabulary: a dictionary mapping tokens to integers.
      tokenizer: a function to use to tokenize each sentence;
        if None, basic_tokenizer will be used.
      normalize_digits: Boolean; if true, all digits are replaced by 0s.
  
    Returns:
      a list of integers, the token-ids for the sentence.
docstring_summary: Convert a string to list of integers representing token-ids.
docstring_tokens: Convert a string to list of integers representing token - ids .
function: def sentence_to_token_ids(sentence, vocabulary,
                            tokenizer=None, normalize_digits=True):
    """Convert a string to list of integers representing token-ids.
  
    For example, a sentence "I have a dog" may become tokenized into
    ["I", "have", "a", "dog"] and with vocabulary {"I": 1, "have": 2,
    "a": 4, "dog": 7"} this function will return [1, 2, 4, 7].
  
    Args:
      sentence: the sentence in bytes format to convert to token-ids.
      vocabulary: a dictionary mapping tokens to integers.
      tokenizer: a function to use to tokenize each sentence;
        if None, basic_tokenizer will be used.
      normalize_digits: Boolean; if true, all digits are replaced by 0s.
  
    Returns:
      a list of integers, the token-ids for the sentence.
    """
  
    if tokenizer:
      words = tokenizer(sentence)
    else:
      words = basic_tokenizer(sentence)
    if not normalize_digits:
      return [vocabulary.get(w, UNK_ID) for w in words]
    # Normalize digits by 0 before looking words up in the vocabulary.
    return [vocabulary.get(_DIGIT_RE.sub(b"0", w), UNK_ID) for w in words]
function_tokens: def sentence_to_token_ids ( sentence , vocabulary , tokenizer = None , normalize_digits = True ) : if tokenizer : words = tokenizer ( sentence ) else : words = basic_tokenizer ( sentence ) if not normalize_digits : return [ vocabulary . get ( w , UNK_ID ) for w in words ] # Normalize digits by 0 before looking words up in the vocabulary. return [ vocabulary . get ( _DIGIT_RE . sub ( b"0" , w ) , UNK_ID ) for w in words ]