def preprocess_utterances(utterances, utterance_dataset):
    utterances = tokenize(utterances)
    utterances = add_start_stop_tokens(utterances)
    tokenized_utterances = copy.deepcopy(utterances)
    utterances = padding(utterances, utterance_dataset.seq_len)
    word2idx = utterance_dataset.word2idx
    utterances = [[word2idx.get(token, word2idx[UNK_TOKEN]) for token in sent] for sent in utterances]
    return (utterances, tokenized_utterances)
def preprocess_utterances(utterances, utterance_dataset):
    utterances = tokenize(utterances)
    utterances = add_start_stop_tokens(utterances)
    tokenized_utterances = copy.deepcopy(utterances)
    utterances = padding(utterances, utterance_dataset.seq_len)
    word2idx = utterance_dataset.word2idx
    utterances = [[word2idx.get(token, word2idx[UNK_TOKEN]) for token in sent] for sent in utterances]
    return (utterances, tokenized_utterances)