<jupyter_start><jupyter_text><jupyter_code>pip install transformers
import transformers
from transformers import AutoTokenizer
# tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-xsum")
tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
tokenized = tokenizer.encode_plus(
    text = "this word is apple",
    text_pair = "this word is not related with sports",
    truncation=True,
    add_special_tokens=True,
    max_length=20,
    pad_to_max_length=True,
    # truncation=True
    # add_special_tokens=True,
    # max_length = 30,
    # padding=True,            #   1)padding='longest' | 2) padding=True  (1,2가 같음) |   3) pad_to_max_length=True   |   4) padding='max_length'
    # return_tensors = 'pt',               # pt : pytorch tensor output    tf : tf tensor output   np : np array output
    # return_length = True
)
tokenized               # token_type_ids : text는 0, text_pair는 1 로 되어있음.
print(tokenizer.convert_ids_to_tokens(tokenized["input_ids"]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1, 'fraction_numerical': 0.011428571428571429, 'mean_word_length': 4.004761904761905, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '76972', 'license': 'MIT', 'n_tokens_mistral': 364, 'n_tokens_neox': 358, 'n_words': 72, 'path': 'nlp/tokenizer_test.ipynb', 'repo_name': 'hihunjin/Code-snippet-for-everything', 'size': '1050'}