


import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin
class  SCREAMING_SNAKE_CASE__  (				lowercase__						,    unittest.TestCase			):
      snake_case__				:     Any        =						BlenderbotSmallTokenizer
      snake_case__				:     str        =						False
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			Any:
             super().setUp()

             a_					:		Tuple							=  ['__start__', 'adapt', 'act', 'ap@@', 'te', '__end__', '__unk__']
             a_					:		Any							=  dict(zip(SCREAMING_SNAKE_CASE__     ,							range(len(SCREAMING_SNAKE_CASE__     )     )     )     )

             a_					:		Optional[Any]							=  ['#version: 0.2', 'a p', 't e</w>', 'ap t</w>', 'a d', 'ad apt</w>', 'a c', 'ac t</w>', '']
             a_					:		str							=  {'unk_token': '__unk__', 'bos_token': '__start__', 'eos_token': '__end__'}

             a_					:		Optional[int]							=  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['vocab_file']     )
             a_					:		Tuple							=  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES['merges_file']     )
             with open(self.vocab_file     ,							'w'     ,							encoding='utf-8'     ) as fp:
                    fp.write(json.dumps(SCREAMING_SNAKE_CASE__     ) + '\n'     )
             with open(self.merges_file     ,							'w'     ,							encoding='utf-8'     ) as fp:
                    fp.write('\n'.join(SCREAMING_SNAKE_CASE__     )     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     ,							**SCREAMING_SNAKE_CASE__				:       Dict     )		->			Dict:
             kwargs.update(self.special_tokens_map     )
             return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname     ,							**SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]     )		->			Union[str, Any]:
             a_					:		Dict							=  'adapt act apte'
             a_					:		List[Any]							=  'adapt act apte'
             return input_text, output_text
      def        SCREAMING_SNAKE_CASE							(       self				:       Tuple     )		->			Optional[Any]:
             a_					:		Any							=  BlenderbotSmallTokenizer(self.vocab_file     ,							self.merges_file     ,							**self.special_tokens_map     )
             a_					:		Optional[int]							=  'adapt act apte'
             a_					:		Optional[int]							=  ['adapt', 'act', 'ap@@', 'te']
             a_					:		Optional[Any]							=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__     )
             self.assertListEqual(SCREAMING_SNAKE_CASE__     ,							SCREAMING_SNAKE_CASE__     )

             a_					:		str							=  [tokenizer.bos_token] + tokens + [tokenizer.eos_token]

             a_					:		str							=  [0, 1, 2, 3, 4, 5]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )     ,							SCREAMING_SNAKE_CASE__     )
      def        SCREAMING_SNAKE_CASE							(       self				:       Optional[Any]     )		->			str:
             a_					:		Any							=  BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'     )
             assert tok('sam'     ).input_ids == [1_3_8_4]
             a_					:		Optional[Any]							=  'I am a small frog.'
             a_					:		Dict							=  tok([src_text]     ,							padding=SCREAMING_SNAKE_CASE__     ,							truncation=SCREAMING_SNAKE_CASE__     )['input_ids']
             a_					:		int							=  tok.batch_decode(SCREAMING_SNAKE_CASE__     ,							skip_special_tokens=SCREAMING_SNAKE_CASE__     ,							clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__     )[0]
             assert src_text != decoded  # I wish it did!
             assert decoded == "i am a small frog ."

      def        SCREAMING_SNAKE_CASE							(       self				:       List[Any]     )		->			str:
             a_					:		Dict							=  BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'     )
             a_					:		int							=  'I am a small frog .'
             a_					:		List[str]							=  '.'
             a_					:		Tuple							=  tok(SCREAMING_SNAKE_CASE__     )['input_ids']
             a_					:		Union[str, Any]							=  tok(SCREAMING_SNAKE_CASE__     )['input_ids']

             assert encoded[-1] == encoded_dot[0]




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__		:List[Any]   =    logging.get_logger(__name__)

lowercase__		:Optional[Any]   =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__		:Union[str, Any]   =    {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__		:str   =    {
    "bert-base-uncased": 512,
    "bert-large-uncased": 512,
    "bert-base-cased": 512,
    "bert-large-cased": 512,
    "bert-base-multilingual-uncased": 512,
    "bert-base-multilingual-cased": 512,
    "bert-base-chinese": 512,
    "bert-base-german-cased": 512,
    "bert-large-uncased-whole-word-masking": 512,
    "bert-large-cased-whole-word-masking": 512,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
    "bert-base-cased-finetuned-mrpc": 512,
    "bert-base-german-dbmdz-cased": 512,
    "bert-base-german-dbmdz-uncased": 512,
    "TurkuNLP/bert-base-finnish-cased-v1": 512,
    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
    "wietsedv/bert-base-dutch-cased": 512,
}

lowercase__		:Dict   =    {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       str       					=VOCAB_FILES_NAMES
							lowercase_       :       Dict       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       Optional[int]       					=PRETRAINED_INIT_CONFIGURATION
							lowercase_       :       Tuple       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Any       					=BertTokenizer




							def __init__(				self   ,A__=None   ,A__=None   ,A__=True   ,A__="[UNK]"   ,A__="[SEP]"   ,A__="[PAD]"   ,A__="[CLS]"   ,A__="[MASK]"   ,A__=True   ,A__=None   ,**A__   ,):
												super().__init__(
												    A__   ,tokenizer_file=A__   ,do_lower_case=A__   ,unk_token=A__   ,sep_token=A__   ,pad_token=A__   ,cls_token=A__   ,mask_token=A__   ,tokenize_chinese_chars=A__   ,strip_accents=A__   ,**A__   ,)

												lowercase							=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
												if (
												    normalizer_state.get('''lowercase'''   ,A__) != do_lower_case
												    or normalizer_state.get('''strip_accents'''   ,A__) != strip_accents
												    or normalizer_state.get('''handle_chinese_chars'''   ,A__) != tokenize_chinese_chars
												):
																	lowercase							=						getattr(A__   ,normalizer_state.pop('''type'''))
																	lowercase							=						do_lower_case
																	lowercase							=						strip_accents
																	lowercase							=						tokenize_chinese_chars
																	lowercase							=						normalizer_class(**A__)

												lowercase							=						do_lower_case




							def  A__      (				self   ,A__   ,A__=None):
												lowercase							=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

												return output




							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						[self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						self._tokenizer.model.save(A__   ,name=A__)
												return tuple(A__)




