def mask_tokens(self, inputs: torch.Tensor, special_tokens_mask: Optional[torch.Tensor]=None) -> Tuple[(torch.Tensor, torch.Tensor)]:
    '\n        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.\n        '
    labels = inputs.clone()
    probability_matrix = torch.full(labels.shape, self.mlm_probability)
    if (special_tokens_mask is None):
        special_tokens_mask = [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()]
        special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool)
    else:
        special_tokens_mask = special_tokens_mask.bool()
    probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
    masked_indices = torch.bernoulli(probability_matrix).bool()
    labels[(~ masked_indices)] = (- 100)
    indices_replaced = (torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices)
    inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
    indices_random = ((torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices) & (~ indices_replaced))
    random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
    inputs[indices_random] = random_words[indices_random]
    return (inputs, labels)
Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.
def mask_tokens(self, inputs: torch.Tensor, special_tokens_mask: Optional[torch.Tensor]=None) -> Tuple[(torch.Tensor, torch.Tensor)]:
    '\n        \n        '
    labels = inputs.clone()
    probability_matrix = torch.full(labels.shape, self.mlm_probability)
    if (special_tokens_mask is None):
        special_tokens_mask = [self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()]
        special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool)
    else:
        special_tokens_mask = special_tokens_mask.bool()
    probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
    masked_indices = torch.bernoulli(probability_matrix).bool()
    labels[(~ masked_indices)] = (- 100)
    indices_replaced = (torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices)
    inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)
    indices_random = ((torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices) & (~ indices_replaced))
    random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)
    inputs[indices_random] = random_words[indices_random]
    return (inputs, labels)