usrbinenv python3 codingutf8 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tests for blenderbot tokenizers including common tests for blenderbotsmalltokenizer import unittest from transformers import blenderbottokenizer blenderbottokenizerfast from transformers testingutils import requirejinja from transformers utils import cachedproperty class blenderbot3btokenizertestsunittest testcase cachedproperty def tokenizer3bself return blenderbottokenizer frompretrainedfacebookblenderbot3b cachedproperty def rusttokenizer3bself return blenderbottokenizerfast frompretrainedfacebookblenderbot3b def testencodedecodecycleself tok self tokenizer3b srctext i am a small frog encoded toksrctext paddingfalse truncationfalseinputids decoded tok batchdecodeencoded skipspecialtokenstrue cleanuptokenizationspacesfalse0 assert srctext decoded def testencodedecodecyclerusttokenizerself tok self rusttokenizer3b srctext i am a small frog encoded toksrctext paddingfalse truncationfalseinputids decoded tok batchdecodeencoded skipspecialtokenstrue cleanuptokenizationspacesfalse0 assert srctext decoded def test3btokenizationsameasparlaiself assert self tokenizer3b addprefixspace assert self tokenizer3b sam sam inputids 5502 2 5502 2 def test3btokenizationsameasparlairusttokenizerself assert self rusttokenizer3b addprefixspace assert self rusttokenizer3b sam sam inputids 5502 2 5502 2 requirejinja def testtokenizationforchatself tok self tokenizer3b testchats role system content you are a helpful chatbot role user content hello role system content you are a helpful chatbot role user content hello role assistant content nice to meet you role assistant content nice to meet you role user content hello tokenizedchats tok applychattemplatetestchat for testchat in testchats expectedtokens 553 366 265 4792 3879 73 311 21 228 228 6950 8 2 553 366 265 4792 3879 73 311 21 228 228 6950 8 228 3490 287 2273 304 21 2 3490 287 2273 304 21 228 228 6950 8 2 for tokenizedchat expectedtokens in ziptokenizedchats expectedtokens self assertlistequaltokenizedchat expectedtokens usr bin env python3 coding utf 8 2020 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tests for blenderbot tokenizers including common tests for blenderbotsmalltokenizer
import unittest

from transformers import BlenderbotTokenizer, BlenderbotTokenizerFast
from transformers.testing_utils import require_jinja
from transformers.utils import cached_property


class Blenderbot3BTokenizerTests(unittest.TestCase):
    @cached_property
    def tokenizer_3b(self):
        return BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B")

    @cached_property
    def rust_tokenizer_3b(self):
        return BlenderbotTokenizerFast.from_pretrained("facebook/blenderbot-3B")

    def test_encode_decode_cycle(self):
        tok = self.tokenizer_3b
        src_text = " I am a small frog."
        encoded = tok([src_text], padding=False, truncation=False)["input_ids"]
        decoded = tok.batch_decode(encoded, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        assert src_text == decoded

    def test_encode_decode_cycle_rust_tokenizer(self):
        tok = self.rust_tokenizer_3b
        src_text = " I am a small frog."
        encoded = tok([src_text], padding=False, truncation=False)["input_ids"]
        decoded = tok.batch_decode(encoded, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        assert src_text == decoded

    def test_3B_tokenization_same_as_parlai(self):
        assert self.tokenizer_3b.add_prefix_space
        assert self.tokenizer_3b([" Sam", "Sam"]).input_ids == [[5502, 2], [5502, 2]]

    def test_3B_tokenization_same_as_parlai_rust_tokenizer(self):
        assert self.rust_tokenizer_3b.add_prefix_space
        assert self.rust_tokenizer_3b([" Sam", "Sam"]).input_ids == [[5502, 2], [5502, 2]]

    @require_jinja
    def test_tokenization_for_chat(self):
        tok = self.tokenizer_3b
        test_chats = [
            [{"role": "system", "content": "You are a helpful chatbot."}, {"role": "user", "content": "Hello!"}],
            [
                {"role": "system", "content": "You are a helpful chatbot."},
                {"role": "user", "content": "Hello!"},
                {"role": "assistant", "content": "Nice to meet you."},
            ],
            [{"role": "assistant", "content": "Nice to meet you."}, {"role": "user", "content": "Hello!"}],
        ]
        tokenized_chats = [tok.apply_chat_template(test_chat) for test_chat in test_chats]
        expected_tokens = [
            [553, 366, 265, 4792, 3879, 73, 311, 21, 228, 228, 6950, 8, 2],
            [553, 366, 265, 4792, 3879, 73, 311, 21, 228, 228, 6950, 8, 228, 3490, 287, 2273, 304, 21, 2],
            [3490, 287, 2273, 304, 21, 228, 228, 6950, 8, 2],
        ]
        for tokenized_chat, expected_tokens in zip(tokenized_chats, expected_tokens):
            self.assertListEqual(tokenized_chat, expected_tokens)