def create_masked_lm_predictions(self, tokens, np_rng, max_ngrams=3, do_whole_word_mask=True, favor_longer_ngram=False, geometric_dist=False):
    'Creates the predictions for the masked LM objective.\n        Note: Tokens here are vocab ids and not text tokens.'
    cand_indexes = []
    token_boundary = ([0] * len(tokens))
    new_tokens = []
    for (i, token) in enumerate(tokens):
        new_tokens.append((token % len(self.tokenizer)))
        if ((token == self.cls_id) or (token == self.sep_id)):
            token_boundary[i] = 1
            continue
        if (do_whole_word_mask and (len(cand_indexes) >= 1) and (not is_start_piece(self.tokenizer._convert_id_to_token(token)))):
            cand_indexes[(- 1)].append(i)
        else:
            cand_indexes.append([i])
            if is_start_piece(self.tokenizer._convert_id_to_token(token)):
                token_boundary[i] = 1
    tokens = new_tokens
    masked_positions = []
    masked_labels = []
    output_tokens = list(tokens)
    if (self.mask_lm_prob == 0):
        return (output_tokens, masked_positions, masked_labels)
    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if ((token == self.cls_id) or (token == self.sep_id)):
            continue
        if (do_whole_word_mask and (len(cand_indexes) >= 1) and (token_boundary[i] == 0)):
            cand_indexes[(- 1)].append(i)
        else:
            cand_indexes.append([i])
    num_to_predict = min(self.max_preds_per_seq, max(1, int(round((len(tokens) * self.mask_lm_prob)))))
    ngrams = np.arange(1, (max_ngrams + 1), dtype=np.int64)
    if (not geometric_dist):
        pvals = (1.0 / np.arange(1, (max_ngrams + 1)))
        pvals /= pvals.sum(keepdims=True)
        if favor_longer_ngram:
            pvals = pvals[::(- 1)]
    ngram_indexes = []
    for idx in range(len(cand_indexes)):
        ngram_index = []
        for n in ngrams:
            ngram_index.append(cand_indexes[idx:(idx + n)])
        ngram_indexes.append(ngram_index)
    np_rng.shuffle(ngram_indexes)
    masked_lms = []
    covered_indexes = set()
    for cand_index_set in ngram_indexes:
        if (len(masked_lms) >= num_to_predict):
            break
        if (not cand_index_set):
            continue
        for index_set in cand_index_set[0]:
            for index in index_set:
                if (index in covered_indexes):
                    continue
        if (not geometric_dist):
            n = np_rng.choice(ngrams[:len(cand_index_set)], p=(pvals[:len(cand_index_set)] / pvals[:len(cand_index_set)].sum(keepdims=True)))
        else:
            n = min(np_rng.geometric(0.2), max_ngrams)
        index_set = sum(cand_index_set[(n - 1)], [])
        n -= 1
        while ((len(masked_lms) + len(index_set)) > num_to_predict):
            if (n == 0):
                break
            index_set = sum(cand_index_set[(n - 1)], [])
            n -= 1
        if ((len(masked_lms) + len(index_set)) > num_to_predict):
            continue
        is_any_index_covered = False
        for index in index_set:
            if (index in covered_indexes):
                is_any_index_covered = True
                break
        if is_any_index_covered:
            continue
        for index in index_set:
            covered_indexes.add(index)
            label = self.mask_token(index, output_tokens, np_rng)
            masked_lms.append(MaskedLmInstance(index=index, label=label))
    masked_lms = sorted(masked_lms, key=(lambda x: x.index))
    for p in masked_lms:
        masked_positions.append(p.index)
        masked_labels.append(p.label)
    return (output_tokens, masked_positions, masked_labels)
Creates the predictions for the masked LM objective.
Note: Tokens here are vocab ids and not text tokens.
def create_masked_lm_predictions(self, tokens, np_rng, max_ngrams=3, do_whole_word_mask=True, favor_longer_ngram=False, geometric_dist=False):
    'Creates the predictions for the masked LM objective.\n        Note: Tokens here are vocab ids and not text tokens.'
    cand_indexes = []
    token_boundary = ([0] * len(tokens))
    new_tokens = []
    for (i, token) in enumerate(tokens):
        new_tokens.append((token % len(self.tokenizer)))
        if ((token == self.cls_id) or (token == self.sep_id)):
            token_boundary[i] = 1
            continue
        if (do_whole_word_mask and (len(cand_indexes) >= 1) and (not is_start_piece(self.tokenizer._convert_id_to_token(token)))):
            cand_indexes[(- 1)].append(i)
        else:
            cand_indexes.append([i])
            if is_start_piece(self.tokenizer._convert_id_to_token(token)):
                token_boundary[i] = 1
    tokens = new_tokens
    masked_positions = []
    masked_labels = []
    output_tokens = list(tokens)
    if (self.mask_lm_prob == 0):
        return (output_tokens, masked_positions, masked_labels)
    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if ((token == self.cls_id) or (token == self.sep_id)):
            continue
        if (do_whole_word_mask and (len(cand_indexes) >= 1) and (token_boundary[i] == 0)):
            cand_indexes[(- 1)].append(i)
        else:
            cand_indexes.append([i])
    num_to_predict = min(self.max_preds_per_seq, max(1, int(round((len(tokens) * self.mask_lm_prob)))))
    ngrams = np.arange(1, (max_ngrams + 1), dtype=np.int64)
    if (not geometric_dist):
        pvals = (1.0 / np.arange(1, (max_ngrams + 1)))
        pvals /= pvals.sum(keepdims=True)
        if favor_longer_ngram:
            pvals = pvals[::(- 1)]
    ngram_indexes = []
    for idx in range(len(cand_indexes)):
        ngram_index = []
        for n in ngrams:
            ngram_index.append(cand_indexes[idx:(idx + n)])
        ngram_indexes.append(ngram_index)
    np_rng.shuffle(ngram_indexes)
    masked_lms = []
    covered_indexes = set()
    for cand_index_set in ngram_indexes:
        if (len(masked_lms) >= num_to_predict):
            break
        if (not cand_index_set):
            continue
        for index_set in cand_index_set[0]:
            for index in index_set:
                if (index in covered_indexes):
                    continue
        if (not geometric_dist):
            n = np_rng.choice(ngrams[:len(cand_index_set)], p=(pvals[:len(cand_index_set)] / pvals[:len(cand_index_set)].sum(keepdims=True)))
        else:
            n = min(np_rng.geometric(0.2), max_ngrams)
        index_set = sum(cand_index_set[(n - 1)], [])
        n -= 1
        while ((len(masked_lms) + len(index_set)) > num_to_predict):
            if (n == 0):
                break
            index_set = sum(cand_index_set[(n - 1)], [])
            n -= 1
        if ((len(masked_lms) + len(index_set)) > num_to_predict):
            continue
        is_any_index_covered = False
        for index in index_set:
            if (index in covered_indexes):
                is_any_index_covered = True
                break
        if is_any_index_covered:
            continue
        for index in index_set:
            covered_indexes.add(index)
            label = self.mask_token(index, output_tokens, np_rng)
            masked_lms.append(MaskedLmInstance(index=index, label=label))
    masked_lms = sorted(masked_lms, key=(lambda x: x.index))
    for p in masked_lms:
        masked_positions.append(p.index)
        masked_labels.append(p.label)
    return (output_tokens, masked_positions, masked_labels)