'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase__  :						Dict					=      logging.get_logger(__name__)

UpperCamelCase__  :						Any					=      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase__  :						Optional[int]					=      {
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase__  :						Any					=      {
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

UpperCamelCase__  :						Union[str, Any]					=      {
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}


class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[Any]   				= VOCAB_FILES_NAMES
      UpperCAmelCase_   :   Optional[Any]   				= PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_   :   List[Any]   				= PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase_   :   str   				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase_   :   Dict   				= BertTokenizer




      def __init__(					self       ,lowerCamelCase_=None       ,lowerCamelCase_=None       ,lowerCamelCase_=True       ,lowerCamelCase_="[UNK]"       ,lowerCamelCase_="[SEP]"       ,lowerCamelCase_="[PAD]"       ,lowerCamelCase_="[CLS]"       ,lowerCamelCase_="[MASK]"       ,lowerCamelCase_=True       ,lowerCamelCase_=None       ,**lowerCamelCase_       ,)      ->      Optional[int]:



         '''simple docstring'''



         super().__init__(
             lowerCamelCase_       ,tokenizer_file=lowerCamelCase_       ,do_lower_case=lowerCamelCase_       ,unk_token=lowerCamelCase_       ,sep_token=lowerCamelCase_       ,pad_token=lowerCamelCase_       ,cls_token=lowerCamelCase_       ,mask_token=lowerCamelCase_       ,tokenize_chinese_chars=lowerCamelCase_       ,strip_accents=lowerCamelCase_       ,**lowerCamelCase_       ,)

         UpperCAmelCase__				:					int 					=	json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get('''lowercase'''       ,lowerCamelCase_				) != do_lower_case
             or normalizer_state.get('''strip_accents'''       ,lowerCamelCase_				) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''       ,lowerCamelCase_				) != tokenize_chinese_chars
         ):
            UpperCAmelCase__				:					List[str] 					=	getattr(lowerCamelCase_       ,normalizer_state.pop('''type'''				)				)
            UpperCAmelCase__				:					Union[str, Any] 					=	do_lower_case
            UpperCAmelCase__				:					List[str] 					=	strip_accents
            UpperCAmelCase__				:					List[Any] 					=	tokenize_chinese_chars
            UpperCAmelCase__				:					Union[str, Any] 					=	normalizer_class(**lowerCamelCase_				)

         UpperCAmelCase__				:					Union[str, Any] 					=	do_lower_case




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_=None				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

         return output




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      List[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	[self.sep_token_id]
         UpperCAmelCase__				:					Any 					=	[self.cls_token_id]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      Tuple[str]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	self._tokenizer.model.save(lowerCamelCase_       ,name=lowerCamelCase_				)
         return tuple(lowerCamelCase_				)





'''simple docstring'''

import warnings


warnings.warn(
    'memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: '
    '`from accelerate import find_executable_batch_size` to avoid this warning.',
    FutureWarning,
)




