





from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand



def _a				(	lowerCamelCase:   Namespace )	->						List[Any]:







      '''simple docstring'''






      return ConvertCommand(
          args.model_type					,				args.tf_checkpoint					,				args.pytorch_dump_output					,				args.config					,				args.finetuning_task_name )


snake_case__					:							List[str]													=     '\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n'







class A_					(			_lowerCamelCase   ):




   @staticmethod
   def _lowerCAmelCase      (_UpperCamelCase	:ArgumentParser )->					List[str]:
         __A          =				parser.add_parser(
             '''convert'''    ,   help='''CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'''    ,   )
         train_parser.add_argument('''--model_type'''    ,   type=_UpperCamelCase    ,   required=_UpperCamelCase    ,   help='''Model\'s type.''' )
         train_parser.add_argument(
             '''--tf_checkpoint'''    ,   type=_UpperCamelCase    ,   required=_UpperCamelCase    ,   help='''TensorFlow checkpoint path or folder.''' )
         train_parser.add_argument(
             '''--pytorch_dump_output'''    ,   type=_UpperCamelCase    ,   required=_UpperCamelCase    ,   help='''Path to the PyTorch saved model output.''' )
         train_parser.add_argument('''--config'''    ,   type=_UpperCamelCase    ,   default=''''''    ,   help='''Configuration file path or folder.''' )
         train_parser.add_argument(
             '''--finetuning_task_name'''    ,   type=_UpperCamelCase    ,   default=_UpperCamelCase    ,   help='''Optional fine-tuning task name if the TF model was a finetuned model.'''    ,   )
         train_parser.set_defaults(func=_UpperCamelCase )




   def __init__(self	:Union[str, Any]    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:str    ,   *_UpperCamelCase	:List[Any]    ,   )->					Optional[int]:
         __A          =				logging.get_logger('''transformers-cli/converting''' )

         self._logger.info(f"""Loading model {model_type}""" )
         __A          =				model_type
         __A          =				tf_checkpoint
         __A          =				pytorch_dump_output
         __A          =				config
         __A          =				finetuning_task_name






   def _lowerCAmelCase      (self	:int )->					Optional[int]:
         if self._model_type == "albert":
               try:
                     from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "bert":
               try:
                     from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "funnel":
               try:
                     from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                         convert_tf_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "t5":
               try:
                     from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_tf_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "gpt":
               from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                   convert_openai_checkpoint_to_pytorch,
               )

               convert_openai_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "transfo_xl":
               try:
                     from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
                         convert_transfo_xl_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               if "ckpt" in self._tf_checkpoint.lower():
                     __A          =				self._tf_checkpoint
                     __A          =				''''''
               else:
                     __A          =				self._tf_checkpoint
                     __A          =				''''''
               convert_transfo_xl_checkpoint_to_pytorch(
                   _UpperCamelCase    ,   self._config    ,   self._pytorch_dump_output    ,   _UpperCamelCase )
         elif self._model_type == "gpt2":
               try:
                     from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
                         convert_gpta_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         elif self._model_type == "xlnet":
               try:
                     from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                         convert_xlnet_checkpoint_to_pytorch,
                     )
               except ImportError:
                     raise ImportError(_UpperCamelCase )

               convert_xlnet_checkpoint_to_pytorch(
                   self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output    ,   self._finetuning_task_name )
         elif self._model_type == "xlm":
               from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                   convert_xlm_checkpoint_to_pytorch,
               )

               convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._pytorch_dump_output )
         elif self._model_type == "lxmert":
               from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
                   convert_lxmert_checkpoint_to_pytorch,
               )

               convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._pytorch_dump_output )
         elif self._model_type == "rembert":
               from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
                   convert_rembert_tf_checkpoint_to_pytorch,
               )

               convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint    ,   self._config    ,   self._pytorch_dump_output )
         else:
               raise ValueError(
                   '''--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]''' )












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


snake_case__					:							List[Any]													=     logging.get_logger(__name__)

snake_case__					:							Union[str, Any]													=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

snake_case__					:							int													=     {
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

snake_case__					:							int													=     {
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

snake_case__					:							str													=     {
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}





class A_					(			_lowerCamelCase   ):
   lowerCAmelCase__        = VOCAB_FILES_NAMES
   lowerCAmelCase__        = PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase__        = PRETRAINED_INIT_CONFIGURATION
   lowerCAmelCase__        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase__        = BertTokenizer




   def __init__(self	:List[str]    ,   _UpperCamelCase	:List[str]=None    ,   _UpperCamelCase	:Optional[Any]=None    ,   _UpperCamelCase	:str=True    ,   _UpperCamelCase	:Optional[Any]="[UNK]"    ,   _UpperCamelCase	:Tuple="[SEP]"    ,   _UpperCamelCase	:List[Any]="[PAD]"    ,   _UpperCamelCase	:int="[CLS]"    ,   _UpperCamelCase	:Optional[int]="[MASK]"    ,   _UpperCamelCase	:Union[str, Any]=True    ,   _UpperCamelCase	:str=None    ,   **_UpperCamelCase	:List[str]    ,   )->					str:
         super().__init__(
             _UpperCamelCase    ,   tokenizer_file=_UpperCamelCase    ,   do_lower_case=_UpperCamelCase    ,   unk_token=_UpperCamelCase    ,   sep_token=_UpperCamelCase    ,   pad_token=_UpperCamelCase    ,   cls_token=_UpperCamelCase    ,   mask_token=_UpperCamelCase    ,   tokenize_chinese_chars=_UpperCamelCase    ,   strip_accents=_UpperCamelCase    ,   **_UpperCamelCase    ,   )

         __A          =				json.loads(self.backend_tokenizer.normalizer.__getstate__() )
         if (
             normalizer_state.get('''lowercase'''    ,   _UpperCamelCase ) != do_lower_case
             or normalizer_state.get('''strip_accents'''    ,   _UpperCamelCase ) != strip_accents
             or normalizer_state.get('''handle_chinese_chars'''    ,   _UpperCamelCase ) != tokenize_chinese_chars
         ):
               __A          =				getattr(_UpperCamelCase    ,   normalizer_state.pop('''type''' ) )
               __A          =				do_lower_case
               __A          =				strip_accents
               __A          =				tokenize_chinese_chars
               __A          =				normalizer_class(**_UpperCamelCase )

         __A          =				do_lower_case




   def _lowerCAmelCase      (self	:Any    ,   _UpperCamelCase	:int    ,   _UpperCamelCase	:List[str]=None )->					List[Any]:
         __A          =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

         return output




   def _lowerCAmelCase      (self	:List[str]    ,   _UpperCamelCase	:List[int]    ,   _UpperCamelCase	:Optional[List[int]] = None )->					List[int]:
         __A          =				[self.sep_token_id]
         __A          =				[self.cls_token_id]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
         return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






   def _lowerCAmelCase      (self	:Any    ,   _UpperCamelCase	:str    ,   _UpperCamelCase	:Optional[str] = None )->					Tuple[str]:
         __A          =				self._tokenizer.model.save(_UpperCamelCase    ,   name=_UpperCamelCase )
         return tuple(_UpperCamelCase )





