<jupyter_start><jupyter_text>Input IDs<jupyter_code>from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
sequence = "A Titan RTX has 24GB of VRAM"
tokenized_sequence = tokenizer.tokenize(sequence)
print(tokenized_sequence)
inputs = tokenizer(sequence)
encoded_sequence = inputs["input_ids"]
print(encoded_sequence)
decoded_sequence = tokenizer.decode(encoded_sequence)
print(decoded_sequence)<jupyter_output>[CLS] A Titan RTX has 24GB of VRAM [SEP]<jupyter_text>Attention mask The attention mask is an optional argument used when batching sequences together. This argument indicates to the model which tokens should be attended to, and which should not.<jupyter_code>from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
sequence_a = "This is a short sequence."
sequence_b = "This is a rather long sequence. It is at least longer than the sequence A."
encoded_sequence_a = tokenizer(sequence_a)["input_ids"]
encoded_sequence_b = tokenizer(sequence_b)["input_ids"]
len(encoded_sequence_a), len(encoded_sequence_b)
padded_sequences = tokenizer([sequence_a, sequence_b], padding=True)
padded_sequences["input_ids"]
padded_sequences["attention_mask"]<jupyter_output><empty_output><jupyter_text>Token Type IDs<jupyter_code>from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-cased")
sequence_a = "HuggingFace is based in NYC"
sequence_b = "Where is HuggingFace based?"
encoded_dict = tokenizer(sequence_a, sequence_b)
decoded = tokenizer.decode(encoded_dict["input_ids"])
print(decoded)
encoded_dict['token_type_ids']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08895522388059701, 'fraction_numerical': 0.0023880597014925373, 'mean_word_length': 9.409937888198758, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '29134', 'license': 'MIT', 'n_tokens_mistral': 534, 'n_tokens_neox': 515, 'n_words': 127, 'path': 'model_input.ipynb', 'repo_name': 'nattiya/Transformers', 'size': '1675'}