


"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase__					=logging.get_logger(__name__)

UpperCAmelCase__					={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__					={
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

UpperCAmelCase__					={
    "junnyu/roformer_chinese_small": 1536,
    "junnyu/roformer_chinese_base": 1536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


UpperCAmelCase__					={
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}





class  lowerCamelCase__    (		_a     ):
      a			:  Tuple           =		VOCAB_FILES_NAMES
      a			:  Union[str, Any]           =		PRETRAINED_VOCAB_FILES_MAP
      a			:  Any           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a			:  Dict           =		PRETRAINED_INIT_CONFIGURATION
      a			:  Optional[Any]           =		RoFormerTokenizer
      def __init__(		self						:						Union[str, Any]   ,							A_						:						List[str]=None   ,							A_						:						Any=None   ,							A_						:						int=True   ,							A_						:						Union[str, Any]="[UNK]"   ,							A_						:						Tuple="[SEP]"   ,							A_						:						Tuple="[PAD]"   ,							A_						:						str="[CLS]"   ,							A_						:						Tuple="[MASK]"   ,							A_						:						str=True   ,							A_						:						Optional[int]=None   ,							**A_						:						List[str]   ,							):






       '''simple docstring'''


       super().__init__(
           A_   ,							tokenizer_file=A_   ,							do_lower_case=A_   ,							unk_token=A_   ,							sep_token=A_   ,							pad_token=A_   ,							cls_token=A_   ,							mask_token=A_   ,							tokenize_chinese_chars=A_   ,							strip_accents=A_   ,							**A_   ,							)

       __lowercase						=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
       if (
           pre_tok_state.get("""lowercase"""   ,							A_				) != do_lower_case
           or pre_tok_state.get("""strip_accents"""   ,							A_				) != strip_accents
       ):
        __lowercase						=						getattr(A_   ,							pre_tok_state.pop("""type"""				)				)
        __lowercase						=						do_lower_case
        __lowercase						=						strip_accents
        __lowercase						=						pre_tok_class(**A_				)

       __lowercase						=						do_lower_case
      def __getstate__(		self						:						Dict				):






       '''simple docstring'''


       __lowercase						=						self.__dict__.copy()
       __lowercase						=						BertPreTokenizer()
       return state
      def __setstate__(		self						:						Union[str, Any]   ,							A_						:						str				):






       '''simple docstring'''


       __lowercase						=						d
       __lowercase						=						self.__dict__["""_tokenizer"""].get_vocab()
       __lowercase						=						PreTokenizer.custom(JiebaPreTokenizer(A_				)				)
      def SCREAMING_SNAKE_CASE_						(		self						:						Union[str, Any]   ,							A_						:						Optional[Any]   ,							A_						:						Dict=None				):






       '''simple docstring'''


       __lowercase						=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output
      def SCREAMING_SNAKE_CASE_						(		self						:						int   ,							A_						:						List[int]   ,							A_						:						Optional[List[int]] = None				):






       '''simple docstring'''


       __lowercase						=						[self.sep_token_id]
       __lowercase						=						[self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep				) * [0]
       return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]
      def SCREAMING_SNAKE_CASE_						(		self						:						List[Any]   ,							A_						:						str   ,							A_						:						Optional[str] = None				):






       '''simple docstring'''


       __lowercase						=						self._tokenizer.model.save(A_   ,							name=A_				)
       return tuple(A_				)






      def SCREAMING_SNAKE_CASE_						(		self						:						List[str]   ,							A_						:						Any   ,							A_						:						List[Any]=None   ,							A_						:						Dict=None   ,							A_						:						Optional[Any]=False   ,							**A_						:						str   ,							):






       '''simple docstring'''


       __lowercase						=						BertPreTokenizer()
       return super().save_pretrained(A_   ,							A_   ,							A_   ,							A_   ,							**A_				)




"""simple docstring"""

def lowerCAmelCase_   (					UpperCamelCase__						:			list					, UpperCamelCase__						:			int					, UpperCamelCase__						:			int = 0					, UpperCamelCase__						:			int = 0			):






 """simple docstring"""







 __lowercase						=						right or len(UpperCamelCase__			) - 1
 if left > right:
  return -1
 elif list_data[left] == key:
  return left
 elif list_data[right] == key:
  return right
 else:
  return search(UpperCamelCase__					, UpperCamelCase__					, left + 1					, right - 1			)


if __name__ == "__main__":
       import doctest

       doctest.testmod()
