
"""simple docstring"""





from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a		       =							logging.get_logger(__name__)


_a		       =							{
    'nielsr/canine-s': 2_048,
}

# Unicode defines 1,114,112 total “codepoints”
_a		       =							1_114_112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_a		       =							0

_a		       =							0xe000
_a		       =							0xe001
_a		       =							0xe002
_a		       =							0xe003
_a		       =							0xe004

# Maps special codepoints to human-readable names.
_a		       =							{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_a		       =							{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}


class A_	(lowercase__      ):




  '''simple docstring'''
  SCREAMING_SNAKE_CASE__			:						List[str]       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







  def __init__(		self				,   lowercase_=chr(lowercase_  )				,   lowercase_=chr(lowercase_  )				,   lowercase_=chr(lowercase_  )				,   lowercase_=chr(lowercase_  )				,   lowercase_=chr(lowercase_  )				,   lowercase_=chr(lowercase_  )				,   lowercase_=False				,   lowercase_=2048				,   **lowercase_				,   ):

     """simple docstring"""
     UpperCAmelCase_					:					Tuple			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else bos_token
     UpperCAmelCase_					:					Union[str, Any]			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else eos_token
     UpperCAmelCase_					:					int			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else sep_token
     UpperCAmelCase_					:					Any			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else cls_token
     UpperCAmelCase_					:					Optional[Any]			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     UpperCAmelCase_					:					Optional[Any]			     =    AddedToken(lowercase_				,   lstrip=lowercase_				,   rstrip=lowercase_  ) if isinstance(lowercase_				,   lowercase_  ) else mask_token

     super().__init__(
         bos_token=lowercase_				,   eos_token=lowercase_				,   sep_token=lowercase_				,   cls_token=lowercase_				,   pad_token=lowercase_				,   mask_token=lowercase_				,   add_prefix_space=lowercase_				,   model_max_length=lowercase_				,   **lowercase_				,   )

     # Creates a mapping for looking up the IDs of special symbols.
     UpperCAmelCase_					:					Dict[str, int]			     =    {}
     for codepoint, name in SPECIAL_CODEPOINTS.items():
        UpperCAmelCase_					:					Union[str, Any]			     =    codepoint

     # Creates a mapping for looking up the string forms of special symbol IDs.
     UpperCAmelCase_					:					Dict[int, str]			     =    {
         codepoint: name for name, codepoint in self._special_codepoints.items()
     }

     UpperCAmelCase_					:					str			     =    UNICODE_VOCAB_SIZE
     UpperCAmelCase_					:					Optional[int]			     =    len(self._special_codepoints  )







  @property
  def        UpperCamelCase__					(		self  ):

     """simple docstring"""
     return self._unicode_vocab_size







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     return list(lowercase_  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     try:
        return ord(lowercase_  )
     except TypeError:
        raise ValueError(F"""invalid token: '{token}'"""  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     try:
        if index in SPECIAL_CODEPOINTS:
           return SPECIAL_CODEPOINTS[index]
        return chr(lowercase_  )
     except TypeError:
        raise ValueError(F"""invalid id: {index}"""  )







  def        UpperCamelCase__					(		self				,   lowercase_  ):

     """simple docstring"""
     return "".join(lowercase_  )







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_ = None  ):

     """simple docstring"""
     UpperCAmelCase_					:					Optional[Any]			     =    [self.sep_token_id]
     UpperCAmelCase_					:					int			     =    [self.cls_token_id]

     UpperCAmelCase_					:					List[Any]			     =    cls + token_ids_a + sep
     if token_ids_a is not None:
        result += token_ids_a + sep
     return result







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_ = None				,   lowercase_ = False  ):

     """simple docstring"""
     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowercase_				,   token_ids_a=lowercase_				,   already_has_special_tokens=lowercase_  )

     UpperCAmelCase_					:					Any			     =    [1] + ([0] * len(lowercase_  )) + [1]
     if token_ids_a is not None:
        result += ([0] * len(lowercase_  )) + [1]
     return result







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_ = None  ):

     """simple docstring"""
     UpperCAmelCase_					:					Tuple			     =    [self.sep_token_id]
     UpperCAmelCase_					:					int			     =    [self.cls_token_id]

     UpperCAmelCase_					:					Union[str, Any]			     =    len(cls + token_ids_a + sep  ) * [0]
     if token_ids_a is not None:
        result += len(token_ids_a + sep  ) * [1]
     return result







  def        UpperCamelCase__					(		self				,   lowercase_				,   lowercase_ = None  ):

     """simple docstring"""
     return ()







"""simple docstring"""

from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)





