codingutf8 2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license can t train newtokenizer via tokenizers lib can t train newtokenizer via tokenizers lib can t serialise custom pretokenizer coding utf 8 2021 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license can t train new_tokenizer via tokenizers lib can t train new_tokenizer via tokenizers lib can t serialise custom pretokenizer
import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_rjieba
@require_tokenizers
class RoFormerTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = RoFormerTokenizer
    rust_tokenizer_class = RoFormerTokenizerFast
    space_between_special_tokens = True
    test_rust_tokenizer = True

    def setUp(self):
        super().setUp()

    def get_tokenizer(self, **kwargs):
        return self.tokenizer_class.from_pretrained("junnyu/roformer_chinese_base", **kwargs)

    def get_rust_tokenizer(self, **kwargs):
        return self.rust_tokenizer_class.from_pretrained("junnyu/roformer_chinese_base", **kwargs)

    def get_chinese_input_output_texts(self):
        input_text = "永和服装饰品有限公司,今天天气非常好"
        output_text = "永和 服装 饰品 有限公司 , 今 天 天 气 非常 好"
        return input_text, output_text

    def test_tokenizer(self):
        tokenizer = self.get_tokenizer()
        input_text, output_text = self.get_chinese_input_output_texts()
        tokens = tokenizer.tokenize(input_text)

        self.assertListEqual(tokens, output_text.split())

        input_tokens = tokens + [tokenizer.unk_token]
        exp_tokens = [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), exp_tokens)

    def test_rust_tokenizer(self):
        tokenizer = self.get_rust_tokenizer()
        input_text, output_text = self.get_chinese_input_output_texts()
        tokens = tokenizer.tokenize(input_text)
        self.assertListEqual(tokens, output_text.split())
        input_tokens = tokens + [tokenizer.unk_token]
        exp_tokens = [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), exp_tokens)

    
    def test_training_new_tokenizer(self):
        pass

    
    def test_training_new_tokenizer_with_special_tokens_change(self):
        pass

    
    def test_save_slow_from_fast_and_reload_fast(self):
        pass