
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase_ 						=						logging.get_logger(__name__)

UpperCamelCase_ 						=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCamelCase_ 						=						{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCamelCase_ 						=						{
    'bert-base-uncased': 5_1_2,
    'bert-large-uncased': 5_1_2,
    'bert-base-cased': 5_1_2,
    'bert-large-cased': 5_1_2,
    'bert-base-multilingual-uncased': 5_1_2,
    'bert-base-multilingual-cased': 5_1_2,
    'bert-base-chinese': 5_1_2,
    'bert-base-german-cased': 5_1_2,
    'bert-large-uncased-whole-word-masking': 5_1_2,
    'bert-large-cased-whole-word-masking': 5_1_2,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-base-cased-finetuned-mrpc': 5_1_2,
    'bert-base-german-dbmdz-cased': 5_1_2,
    'bert-base-german-dbmdz-uncased': 5_1_2,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_1_2,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_1_2,
    'wietsedv/bert-base-dutch-cased': 5_1_2,
}

UpperCamelCase_ 						=						{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}

class    a		( __UpperCAmelCase       ):
							lowercase_		:    List[Any] 				=	VOCAB_FILES_NAMES
							lowercase_		:    Optional[Any] 				=	PRETRAINED_VOCAB_FILES_MAP
							lowercase_		:    List[Any] 				=	PRETRAINED_INIT_CONFIGURATION
							lowercase_		:    str 				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_		:    Dict 				=	BertTokenizer







							def __init__(    self	:	List[str]   ,					snake_case__	:	Optional[int]=None   ,					snake_case__	:	Union[str, Any]=None   ,					snake_case__	:	Any=True   ,					snake_case__	:	Optional[Any]="[UNK]"   ,					snake_case__	:	Optional[int]="[SEP]"   ,					snake_case__	:	Dict="[PAD]"   ,					snake_case__	:	Optional[int]="[CLS]"   ,					snake_case__	:	List[Any]="[MASK]"   ,					snake_case__	:	Optional[int]=True   ,					snake_case__	:	Union[str, Any]=None   ,					**snake_case__	:	Tuple   ,					):
								"""simple docstring"""







								super().__init__(
								    lowerCamelCase_   ,					tokenizer_file=lowerCamelCase_   ,					do_lower_case=lowerCamelCase_   ,					unk_token=lowerCamelCase_   ,					sep_token=lowerCamelCase_   ,					pad_token=lowerCamelCase_   ,					cls_token=lowerCamelCase_   ,					mask_token=lowerCamelCase_   ,					tokenize_chinese_chars=lowerCamelCase_   ,					strip_accents=lowerCamelCase_   ,					**lowerCamelCase_   ,					)

								__lowerCAmelCase    =				json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
								if (
								    normalizer_state.get("lowercase"   ,					lowerCamelCase_			) != do_lower_case
								    or normalizer_state.get("strip_accents"   ,					lowerCamelCase_			) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"   ,					lowerCamelCase_			) != tokenize_chinese_chars
								):
									__lowerCAmelCase    =				getattr(lowerCamelCase_   ,					normalizer_state.pop("type"			)			)
									__lowerCAmelCase    =				do_lower_case
									__lowerCAmelCase    =				strip_accents
									__lowerCAmelCase    =				tokenize_chinese_chars
									__lowerCAmelCase    =				normalizer_class(**lowerCamelCase_			)

								__lowerCAmelCase    =				do_lower_case







							def    UpperCAmelCase__       (    self	:	Union[str, Any]   ,					snake_case__	:	Union[str, Any]   ,					snake_case__	:	Tuple=None			):
								"""simple docstring"""







								__lowerCAmelCase    =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output







							def    UpperCAmelCase__       (    self	:	Optional[Any]   ,					snake_case__	:	List[Any]   ,					snake_case__	:	str = None			):
								"""simple docstring"""







								__lowerCAmelCase    =				[self.sep_token_id]
								__lowerCAmelCase    =				[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep			) * [0]
								return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







							def    UpperCAmelCase__       (    self	:	Tuple   ,					snake_case__	:	Tuple   ,					snake_case__	:	int = None			):
								"""simple docstring"""







								__lowerCAmelCase    =				self._tokenizer.model.save(lowerCamelCase_   ,					name=lowerCamelCase_			)
								return tuple(lowerCamelCase_			)







'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase__  :						List[str]					=      logging.get_logger(__name__)

UpperCamelCase__  :						str					=      {
    'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/config.json',
    'umberto-commoncrawl-cased-v1': (
        'https://huggingface.co/Musixmatch/umberto-commoncrawl-cased-v1/resolve/main/config.json'
    ),
    'umberto-wikipedia-uncased-v1': (
        'https://huggingface.co/Musixmatch/umberto-wikipedia-uncased-v1/resolve/main/config.json'
    ),
}


class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      UpperCAmelCase_   :   Dict   				= '''camembert'''




      def __init__(					self       ,lowerCamelCase_=30522       ,lowerCamelCase_=768       ,lowerCamelCase_=12       ,lowerCamelCase_=12       ,lowerCamelCase_=3072       ,lowerCamelCase_="gelu"       ,lowerCamelCase_=0.1       ,lowerCamelCase_=0.1       ,lowerCamelCase_=512       ,lowerCamelCase_=2       ,lowerCamelCase_=0.02       ,lowerCamelCase_=1e-12       ,lowerCamelCase_=1       ,lowerCamelCase_=0       ,lowerCamelCase_=2       ,lowerCamelCase_="absolute"       ,lowerCamelCase_=True       ,lowerCamelCase_=None       ,**lowerCamelCase_       ,)      ->      str:



         '''simple docstring'''



         super().__init__(pad_token_id=lowerCamelCase_       ,bos_token_id=lowerCamelCase_       ,eos_token_id=lowerCamelCase_       ,**lowerCamelCase_				)

         UpperCAmelCase__				:					int 					=	vocab_size
         UpperCAmelCase__				:					List[Any] 					=	hidden_size
         UpperCAmelCase__				:					Tuple 					=	num_hidden_layers
         UpperCAmelCase__				:					List[str] 					=	num_attention_heads
         UpperCAmelCase__				:					int 					=	hidden_act
         UpperCAmelCase__				:					Tuple 					=	intermediate_size
         UpperCAmelCase__				:					List[Any] 					=	hidden_dropout_prob
         UpperCAmelCase__				:					Tuple 					=	attention_probs_dropout_prob
         UpperCAmelCase__				:					str 					=	max_position_embeddings
         UpperCAmelCase__				:					Union[str, Any] 					=	type_vocab_size
         UpperCAmelCase__				:					int 					=	initializer_range
         UpperCAmelCase__				:					Optional[Any] 					=	layer_norm_eps
         UpperCAmelCase__				:					Optional[Any] 					=	position_embedding_type
         UpperCAmelCase__				:					str 					=	use_cache
         UpperCAmelCase__				:					List[Any] 					=	classifier_dropout




class 					_lowercase						(   lowerCAmelCase ):
      '''simple docstring'''


      @property
      def 		lowerCAmelCase__       (					self				)      ->      Mapping[str, Mapping[int, str]]:



         '''simple docstring'''



         if self.task == "multiple-choice":
            UpperCAmelCase__				:					Tuple 					=	{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
         else:
            UpperCAmelCase__				:					Union[str, Any] 					=	{0: '''batch''', 1: '''sequence'''}
         return OrderedDict(
             [
                 ('''input_ids''', dynamic_axis),
                 ('''attention_mask''', dynamic_axis),
             ]				)




