
"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


UpperCamelCase	:				Optional[Any]  	= logging.get_logger(__name__)

UpperCamelCase	:				Tuple  	= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase	:				List[Any]  	= {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase	:				Optional[Any]  	= {
    '''bert-base-uncased''': 5_1_2,
    '''bert-large-uncased''': 5_1_2,
    '''bert-base-cased''': 5_1_2,
    '''bert-large-cased''': 5_1_2,
    '''bert-base-multilingual-uncased''': 5_1_2,
    '''bert-base-multilingual-cased''': 5_1_2,
    '''bert-base-chinese''': 5_1_2,
    '''bert-base-german-cased''': 5_1_2,
    '''bert-large-uncased-whole-word-masking''': 5_1_2,
    '''bert-large-cased-whole-word-masking''': 5_1_2,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_1_2,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_1_2,
    '''bert-base-cased-finetuned-mrpc''': 5_1_2,
    '''bert-base-german-dbmdz-cased''': 5_1_2,
    '''bert-base-german-dbmdz-uncased''': 5_1_2,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_1_2,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_1_2,
    '''wietsedv/bert-base-dutch-cased''': 5_1_2,
}

UpperCamelCase	:				List[str]  	= {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}

class __lowerCAmelCase		(      UpperCAmelCase__			):
     lowercase						   =						VOCAB_FILES_NAMES
     lowercase						   =						PRETRAINED_VOCAB_FILES_MAP
     lowercase						   =						PRETRAINED_INIT_CONFIGURATION
     lowercase						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase						   =						BertTokenizer





     def __init__(	self	,     __UpperCAmelCase=None	,     __UpperCAmelCase=None	,     __UpperCAmelCase=True	,     __UpperCAmelCase="[UNK]"	,     __UpperCAmelCase="[SEP]"	,     __UpperCAmelCase="[PAD]"	,     __UpperCAmelCase="[CLS]"	,     __UpperCAmelCase="[MASK]"	,     __UpperCAmelCase=True	,     __UpperCAmelCase=None	,     **__UpperCAmelCase	,     ):

      '''simple docstring'''




      super().__init__(
          __UpperCAmelCase	,     tokenizer_file=__UpperCAmelCase	,     do_lower_case=__UpperCAmelCase	,     unk_token=__UpperCAmelCase	,     sep_token=__UpperCAmelCase	,     pad_token=__UpperCAmelCase	,     cls_token=__UpperCAmelCase	,     mask_token=__UpperCAmelCase	,     tokenize_chinese_chars=__UpperCAmelCase	,     strip_accents=__UpperCAmelCase	,     **__UpperCAmelCase	,     )

      __UpperCamelCase				  =       json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
      if (
          normalizer_state.get('lowercase'	,     __UpperCAmelCase      ) != do_lower_case
          or normalizer_state.get('strip_accents'	,     __UpperCAmelCase      ) != strip_accents
          or normalizer_state.get('handle_chinese_chars'	,     __UpperCAmelCase      ) != tokenize_chinese_chars
      ):
       __UpperCamelCase				  =       getattr(__UpperCAmelCase	,     normalizer_state.pop('type'      )      )
       __UpperCamelCase				  =       do_lower_case
       __UpperCamelCase				  =       strip_accents
       __UpperCamelCase				  =       tokenize_chinese_chars
       __UpperCamelCase				  =       normalizer_class(**__UpperCAmelCase      )

      __UpperCamelCase				  =       do_lower_case





     def 						UpperCAmelCase					(	self	,     __UpperCAmelCase	,     __UpperCAmelCase=None      ):

      '''simple docstring'''




      __UpperCamelCase				  =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output





     def 						UpperCAmelCase					(	self	,     __UpperCAmelCase	,     __UpperCAmelCase = None      ):

      '''simple docstring'''




      __UpperCamelCase				  =       [self.sep_token_id]
      __UpperCamelCase				  =       [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







     def 						UpperCAmelCase					(	self	,     __UpperCAmelCase	,     __UpperCAmelCase = None      ):

      '''simple docstring'''




      __UpperCamelCase				  =       self._tokenizer.model.save(__UpperCAmelCase	,     name=__UpperCAmelCase      )
      return tuple(__UpperCAmelCase      )







'''simple docstring'''







A__:    Tuple					    =       '''
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

A__:    Tuple					    =       [{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
A__:    Any					    =       {
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}


