def tokenize(tokenizer, text_array, max_seq_len=64, pad_to_max_length=True, add_special_tokens=True):
    ' Returns tokenized IDs and attention mask\n    The transformers encode_plus method returns the following:\n    {\n    input_ids: list[int],\n    token_type_ids: list[int] if return_token_type_ids is True (default)\n    attention_mask: list[int] if return_attention_mask is True (default)\n    overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True\n    num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True\n    special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True\n    }'
    all_tokens = []
    all_attention_mask = []
    for (i, text) in enumerate(tqdm(text_array)):
        encoded = tokenizer.encode_plus(text, add_special_tokens=add_special_tokens, max_length=max_seq_len, pad_to_max_length=pad_to_max_length)
        tokens = torch.tensor(encoded['input_ids'])
        attention_mask = torch.tensor(encoded['attention_mask'])
        all_tokens.append(tokens)
        all_attention_mask.append(attention_mask)
    return (all_tokens, all_attention_mask)
Returns tokenized IDs and attention mask
The transformers encode_plus method returns the following:
{
input_ids: list[int],
token_type_ids: list[int] if return_token_type_ids is True (default)
attention_mask: list[int] if return_attention_mask is True (default)
overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True
num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True
special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True
}
def tokenize(tokenizer, text_array, max_seq_len=64, pad_to_max_length=True, add_special_tokens=True):
    ' Returns tokenized IDs and attention mask\n    The transformers encode_plus method returns the following:\n    {\n    input_ids: list[int],\n    token_type_ids: list[int] if return_token_type_ids is True (default)\n    attention_mask: list[int] if return_attention_mask is True (default)\n    overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True\n    num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True\n    special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True\n    }'
    all_tokens = []
    all_attention_mask = []
    for (i, text) in enumerate(tqdm(text_array)):
        encoded = tokenizer.encode_plus(text, add_special_tokens=add_special_tokens, max_length=max_seq_len, pad_to_max_length=pad_to_max_length)
        tokens = torch.tensor(encoded['input_ids'])
        attention_mask = torch.tensor(encoded['attention_mask'])
        all_tokens.append(tokens)
        all_attention_mask.append(attention_mask)
    return (all_tokens, all_attention_mask)