





"""simple docstring"""





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A											=		logging.get_logger(__name__)
__A											=		{"""vocab_file""": """spiece.model"""}

__A											=		{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

__A											=		{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

__A											=		"""▁"""






class 					_lowerCAmelCase   (     a       ):
       """simple docstring"""




       __magic_name__     :int		  =					VOCAB_FILES_NAMES
       __magic_name__     :Any		  =					PRETRAINED_VOCAB_FILES_MAP
       __magic_name__     :List[Any]		  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






       def __init__(    self    , __UpperCAmelCase    , __UpperCAmelCase=True    , __UpperCAmelCase=True    , __UpperCAmelCase=False    , __UpperCAmelCase="[CLS]"    , __UpperCAmelCase="[SEP]"    , __UpperCAmelCase="<unk>"    , __UpperCAmelCase="[SEP]"    , __UpperCAmelCase="<pad>"    , __UpperCAmelCase="[CLS]"    , __UpperCAmelCase="[MASK]"    , __UpperCAmelCase = None    , **__UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							(
             AddedToken(__UpperCAmelCase    , lstrip=__UpperCAmelCase    , rstrip=__UpperCAmelCase    , normalized=__UpperCAmelCase      )
             if isinstance(__UpperCAmelCase    , __UpperCAmelCase      )
             else mask_token
         )

         lowerCAmelCase__			:Union[str, Any]											=							{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=__UpperCAmelCase    , remove_space=__UpperCAmelCase    , keep_accents=__UpperCAmelCase    , bos_token=__UpperCAmelCase    , eos_token=__UpperCAmelCase    , unk_token=__UpperCAmelCase    , sep_token=__UpperCAmelCase    , pad_token=__UpperCAmelCase    , cls_token=__UpperCAmelCase    , mask_token=__UpperCAmelCase    , sp_model_kwargs=self.sp_model_kwargs    , **__UpperCAmelCase    , )

         lowerCAmelCase__			:Tuple											=							do_lower_case
         lowerCAmelCase__			:Tuple											=							remove_space
         lowerCAmelCase__			:Optional[Any]											=							keep_accents
         lowerCAmelCase__			:List[str]											=							vocab_file

         lowerCAmelCase__			:List[Any]											=							spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(__UpperCAmelCase      )






       @property
       def      snake_case						(    self      ):



         '''simple docstring'''

         return len(self.sp_model      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							{self.convert_ids_to_tokens(__UpperCAmelCase      ): i for i in range(self.vocab_size      )}
         vocab.update(self.added_tokens_encoder      )
         return vocab






       def __getstate__(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Dict											=							self.__dict__.copy()
         lowerCAmelCase__			:Tuple											=							None
         return state






       def __setstate__(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[str]											=							d

         # for backward compatibility
         if not hasattr(self    , 'sp_model_kwargs'      ):
           lowerCAmelCase__			:Tuple											=							{}

         lowerCAmelCase__			:List[str]											=							spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(self.vocab_file      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         if self.remove_space:
           lowerCAmelCase__			:Union[str, Any]											=							' '.join(inputs.strip().split()      )
         else:
           lowerCAmelCase__			:Union[str, Any]											=							inputs
         lowerCAmelCase__			:List[str]											=							outputs.replace('``'    , '"'      ).replace('\'\''    , '"'      )

         if not self.keep_accents:
           lowerCAmelCase__			:Tuple											=							unicodedata.normalize('NFKD'    , __UpperCAmelCase      )
           lowerCAmelCase__			:Any											=							''.join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase      )]      )
         if self.do_lower_case:
           lowerCAmelCase__			:Dict											=							outputs.lower()

         return outputs






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							self.preprocess_text(__UpperCAmelCase      )
         lowerCAmelCase__			:List[Any]											=							self.sp_model.encode(__UpperCAmelCase    , out_type=__UpperCAmelCase      )
         lowerCAmelCase__			:int											=							[]
         for piece in pieces:
           if len(__UpperCAmelCase      ) > 1 and piece[-1] == str(','      ) and piece[-2].isdigit():
             lowerCAmelCase__			:Any											=							self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase    , ''      )      )
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
               if len(cur_pieces[0]      ) == 1:
                 lowerCAmelCase__			:Dict											=							cur_pieces[1:]
               else:
                 lowerCAmelCase__			:str											=							cur_pieces[0][1:]
             cur_pieces.append(piece[-1]      )
             new_pieces.extend(__UpperCAmelCase      )
           else:
             new_pieces.append(__UpperCAmelCase      )

         return new_pieces






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         return self.sp_model.PieceToId(__UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         return self.sp_model.IdToPiece(__UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							[]
         lowerCAmelCase__			:Dict											=							''
         lowerCAmelCase__			:str											=							False
         for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
             if not prev_is_special:
               out_string += " "
             out_string += self.sp_model.decode(__UpperCAmelCase      ) + token
             lowerCAmelCase__			:List[str]											=							True
             lowerCAmelCase__			:int											=							[]
           else:
             current_sub_tokens.append(__UpperCAmelCase      )
             lowerCAmelCase__			:Optional[Any]											=							False
         out_string += self.sp_model.decode(__UpperCAmelCase      )
         return out_string.strip()






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         lowerCAmelCase__			:int											=							[self.sep_token_id]
         lowerCAmelCase__			:List[Any]											=							[self.cls_token_id]
         if token_ids_a is None:
           return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None    , __UpperCAmelCase = False      ):



         '''simple docstring'''

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__UpperCAmelCase    , token_ids_a=__UpperCAmelCase    , already_has_special_tokens=__UpperCAmelCase      )

         if token_ids_a is not None:
           return [1] + ([0] * len(__UpperCAmelCase      )) + [1] + ([0] * len(__UpperCAmelCase      )) + [1]
         return [1] + ([0] * len(__UpperCAmelCase      )) + [1]






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         lowerCAmelCase__			:Dict											=							[self.sep_token_id]
         lowerCAmelCase__			:Any											=							[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         if not os.path.isdir(__UpperCAmelCase      ):
           logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
           return
         lowerCAmelCase__			:List[str]											=							os.path.join(
             __UpperCAmelCase    , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCAmelCase      ) and os.path.isfile(self.vocab_file      ):
           copyfile(self.vocab_file    , __UpperCAmelCase      )
         elif not os.path.isfile(self.vocab_file      ):
           with open(__UpperCAmelCase    , 'wb'      ) as fi:
             lowerCAmelCase__			:Tuple											=							self.sp_model.serialized_model_proto()
             fi.write(__UpperCAmelCase      )

         return (out_vocab_file,)







"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


__A											=		{
    """configuration_owlvit""": [
        """OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """OwlViTConfig""",
        """OwlViTOnnxConfig""",
        """OwlViTTextConfig""",
        """OwlViTVisionConfig""",
    ],
    """processing_owlvit""": ["""OwlViTProcessor"""],
}


try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		["""OwlViTFeatureExtractor"""]
  __A											=		["""OwlViTImageProcessor"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		[
      """OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """OwlViTModel""",
      """OwlViTPreTrainedModel""",
      """OwlViTTextModel""",
      """OwlViTVisionModel""",
      """OwlViTForObjectDetection""",
  ]

if TYPE_CHECKING:
  from .configuration_owlvit import (
      OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
      OwlViTConfig,
      OwlViTOnnxConfig,
      OwlViTTextConfig,
      OwlViTVisionConfig,
  )
  from .processing_owlvit import OwlViTProcessor

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_owlvit import OwlViTFeatureExtractor
    from .image_processing_owlvit import OwlViTImageProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_owlvit import (
        OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
        OwlViTForObjectDetection,
        OwlViTModel,
        OwlViTPreTrainedModel,
        OwlViTTextModel,
        OwlViTVisionModel,
    )

else:
  import sys

  __A											=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
