<jupyter_start><jupyter_text>Non-overlapping windows<jupyter_code>s1 = ["question question"]
s2 = ["one two three four five six"]
for name, tokenizer in tokenizers.items():
    print(f"========\n{name}")
    x = tokenizer(
        s1, 
        s2, 
        truncation="only_second",
        max_length=10,
        padding="max_length",
        stride=0,
        return_overflowing_tokens=True,
        return_token_type_ids=True,
        return_special_tokens_mask=True,
    )
    print(x.keys())
    for input_ids in x["input_ids"]:
        print(tokenizer.convert_ids_to_tokens(input_ids))
    special_tokens_mask = x.pop("special_tokens_mask")
    print(f"special_tokens_mask={repr(special_tokens_mask)}")
    token_type_ids = x.pop("token_type_ids")
    print(f"token_type_ids={repr(token_type_ids)}")
    overflow_to_sample_mapping = x.pop("overflow_to_sample_mapping")
    print(f"{len(overflow_to_sample_mapping)} overflow_to_sample_mapping={repr(overflow_to_sample_mapping)}")<jupyter_output>========
wordpiece
dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'overflow_to_sample_mapping'])
['[CLS]', 'question', 'question', '[SEP]', 'one', 'two', 'three', 'four', 'five', '[SEP]']
['[CLS]', 'question', 'question', '[SEP]', 'six', '[SEP]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']
special_tokens_mask=[[1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 1, 1, 1, 1, 1]]
token_type_ids=[[0, 0, 0, 0, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 1, 1, 0, 0, 0, 0]]
2 overflow_to_sample_mapping=[0, 0]
========
sentencepiece
dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'overflow_to_sample_mapping'])
['<s>', '▁question', '▁question', '</s>', '</s>', '▁one', '▁two', '▁three', '▁four', '</s>']
['<s>', '▁question', '▁question', '</s>', '</s>', '▁five', '▁six', '</s>', '<pad>', '<pad>']
special_tokens_mask=[[1, 0, 0, 1, 1, 0, 0, 0, 0, 1], [1, 0, 0, 1, 1, 0, 0, 1, 1, 1]]
token_type_ids=[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0,[...]<jupyter_text>Overlapping windows<jupyter_code>s1 = ["question question"]
s2 = ["one two three four five six seven eight nine"]
for name, tokenizer in tokenizers.items():
    print(f"========\n{name}")
    x = tokenizer(
        s1, 
        s2, 
        truncation="only_second",
        max_length=10,
        padding="max_length",
        stride=2,
        return_overflowing_tokens=True,
        return_token_type_ids=True,
        return_special_tokens_mask=True,
    )
    print(x.keys())
    for input_ids in x["input_ids"]:
        print(tokenizer.convert_ids_to_tokens(input_ids))
    special_tokens_mask = x.pop("special_tokens_mask")
    print(f"special_tokens_mask={repr(special_tokens_mask)}")
    token_type_ids = x.pop("token_type_ids")
    print(f"token_type_ids={repr(token_type_ids)}")
    overflow_to_sample_mapping = x.pop("overflow_to_sample_mapping")
    print(f"{len(overflow_to_sample_mapping)} overflow_to_sample_mapping={repr(overflow_to_sample_mapping)}")<jupyter_output>========
wordpiece
dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'overflow_to_sample_mapping'])
['[CLS]', 'question', 'question', '[SEP]', 'one', 'two', 'three', 'four', 'five', '[SEP]']
['[CLS]', 'question', 'question', '[SEP]', 'four', 'five', 'six', 'seven', 'eight', '[SEP]']
['[CLS]', 'question', 'question', '[SEP]', 'seven', 'eight', 'nine', '[SEP]', '[PAD]', '[PAD]']
special_tokens_mask=[[1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 0, 0, 0, 0, 1], [1, 0, 0, 1, 0, 0, 0, 1, 1, 1]]
token_type_ids=[[0, 0, 0, 0, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 1, 1, 1, 1, 0, 0]]
3 overflow_to_sample_mapping=[0, 0, 0]
========
sentencepiece
dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'special_tokens_mask', 'overflow_to_sample_mapping'])
['<s>', '▁question', '▁question', '</s>', '</s>', '▁one', '▁two', '▁three', '▁four', '</s>']
['<s>', '▁question', '▁question', '</s>', '</s>', '▁three', '▁four', '▁five', '▁six', '</[...]
{'fraction_non_alphanumeric': 0.2337824351297405, 'fraction_numerical': 0.03967065868263473, 'mean_word_length': 5.167692307692308, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '598584', 'license': 'MIT', 'n_tokens_mistral': 1687, 'n_tokens_neox': 1602, 'n_words': 348, 'path': 'tokenizer_overflow.ipynb', 'repo_name': 'seahrh/nlp-examples', 'size': '4008'}