



"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowerCamelCase  			=		logging.get_logger(__name__)

lowerCamelCase  			=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase  			=		{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

lowerCamelCase  			=		{
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


lowerCamelCase  			=		{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}
class  lowercase__  (       SCREAMING_SNAKE_CASE ):







       '''simple docstring'''


       UpperCamelCase		=	VOCAB_FILES_NAMES
       UpperCamelCase		=	PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase		=	PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase		=	RoFormerTokenizer



       def __init__(				self     :    Optional[int]						,							_UpperCAmelCase     :    Dict=None						,							_UpperCAmelCase     :    Dict=None						,							_UpperCAmelCase     :    Optional[int]=True						,							_UpperCAmelCase     :    str="[UNK]"						,							_UpperCAmelCase     :    Tuple="[SEP]"						,							_UpperCAmelCase     :    Dict="[PAD]"						,							_UpperCAmelCase     :    Any="[CLS]"						,							_UpperCAmelCase     :    Optional[int]="[MASK]"						,							_UpperCAmelCase     :    List[str]=True						,							_UpperCAmelCase     :    str=None						,							**_UpperCAmelCase     :    Optional[int]						,							)					->		Any:


             '''simple docstring'''




             super().__init__(
                 lowerCamelCase_						,							tokenizer_file=lowerCamelCase_						,							do_lower_case=lowerCamelCase_						,							unk_token=lowerCamelCase_						,							sep_token=lowerCamelCase_						,							pad_token=lowerCamelCase_						,							cls_token=lowerCamelCase_						,							mask_token=lowerCamelCase_						,							tokenize_chinese_chars=lowerCamelCase_						,							strip_accents=lowerCamelCase_						,							**lowerCamelCase_						,							)

             UpperCAmelCase_     					=				json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
             if (
                 pre_tok_state.get("lowercase"						,							lowerCamelCase_       ) != do_lower_case
                 or pre_tok_state.get("strip_accents"						,							lowerCamelCase_       ) != strip_accents
             ):
                   UpperCAmelCase_     					=				getattr(lowerCamelCase_						,							pre_tok_state.pop("type"       )       )
                   UpperCAmelCase_     					=				do_lower_case
                   UpperCAmelCase_     					=				strip_accents
                   UpperCAmelCase_     					=				pre_tok_class(**lowerCamelCase_       )

             UpperCAmelCase_     					=				do_lower_case



       def __getstate__(				self     :    Tuple       )					->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_     					=				self.__dict__.copy()
             UpperCAmelCase_     					=				BertPreTokenizer()
             return state



       def __setstate__(				self     :    Any						,							_UpperCAmelCase     :    Optional[Any]       )					->		str:


             '''simple docstring'''




             UpperCAmelCase_     					=				d
             UpperCAmelCase_     					=				self.__dict__['''_tokenizer'''].get_vocab()
             UpperCAmelCase_     					=				PreTokenizer.custom(JiebaPreTokenizer(lowerCamelCase_       )       )



       def 	lowercase__							(				self     :    Any						,							_UpperCAmelCase     :    int						,							_UpperCAmelCase     :    Union[str, Any]=None       )					->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output



       def 	lowercase__							(				self     :    List[Any]						,							_UpperCAmelCase     :    Tuple						,							_UpperCAmelCase     :    List[Any] = None       )					->		List[int]:


             '''simple docstring'''




             UpperCAmelCase_     					=				[self.sep_token_id]
             UpperCAmelCase_     					=				[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls + token_ids_a + sep       ) * [0]
             return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



       def 	lowercase__							(				self     :    str						,							_UpperCAmelCase     :    Any						,							_UpperCAmelCase     :    Any = None       )					->		Tuple[str]:


             '''simple docstring'''




             UpperCAmelCase_     					=				self._tokenizer.model.save(lowerCamelCase_						,							name=lowerCamelCase_       )
             return tuple(lowerCamelCase_       )







       def 	lowercase__							(				self     :    int						,							_UpperCAmelCase     :    Dict						,							_UpperCAmelCase     :    Union[str, Any]=None						,							_UpperCAmelCase     :    Tuple=None						,							_UpperCAmelCase     :    int=False						,							**_UpperCAmelCase     :    Union[str, Any]						,							)					->		Optional[Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				BertPreTokenizer()
             return super().save_pretrained(lowerCamelCase_						,							lowerCamelCase_						,							lowerCamelCase_						,							lowerCamelCase_						,							**lowerCamelCase_       )





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase__  :						str					=      {
    'configuration_git': ['GIT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GitConfig', 'GitVisionConfig'],
    'processing_git': ['GitProcessor'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__  :						str					=      [
      'GIT_PRETRAINED_MODEL_ARCHIVE_LIST',
      'GitForCausalLM',
      'GitModel',
      'GitPreTrainedModel',
      'GitVisionModel',
  ]

if TYPE_CHECKING:
  from .configuration_git import GIT_PRETRAINED_CONFIG_ARCHIVE_MAP, GitConfig, GitVisionConfig
  from .processing_git import GitProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_git import (
        GIT_PRETRAINED_MODEL_ARCHIVE_LIST,
        GitForCausalLM,
        GitModel,
        GitPreTrainedModel,
        GitVisionModel,
    )

else:
  import sys

  UpperCamelCase__  :						Tuple					=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




