

'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCAmelCase					:			Union[str, Any]       =	logging.get_logger(__name__)

UpperCAmelCase					:			List[str]       =	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase					:			Union[str, Any]       =	{
    'vocab_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase					:			Union[str, Any]       =	{
    'yjernite/retribert-base-uncased': 5_1_2,
}


UpperCAmelCase					:			str       =	{
    'yjernite/retribert-base-uncased': {'do_lower_case': True},
}







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 lowerCAmelCase__       = VOCAB_FILES_NAMES
 lowerCAmelCase__       = PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       = PRETRAINED_INIT_CONFIGURATION
 lowerCAmelCase__       = RetriBertTokenizer
 lowerCAmelCase__       = ["input_ids", "attention_mask"]
 def __init__(						self   :			Any   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=None   ,				__SCREAMING_SNAKE_CASE   :			Optional[int]=None   ,				__SCREAMING_SNAKE_CASE   :			List[str]=True   ,				__SCREAMING_SNAKE_CASE   :			Dict="[UNK]"   ,				__SCREAMING_SNAKE_CASE   :			List[str]="[SEP]"   ,				__SCREAMING_SNAKE_CASE   :			List[Any]="[PAD]"   ,				__SCREAMING_SNAKE_CASE   :			List[Any]="[CLS]"   ,				__SCREAMING_SNAKE_CASE   :			Dict="[MASK]"   ,				__SCREAMING_SNAKE_CASE   :			Dict=True   ,				__SCREAMING_SNAKE_CASE   :			List[str]=None   ,				**__SCREAMING_SNAKE_CASE   :			Optional[Any]   ,				)    ->					Tuple:


      """simple docstring"""




      super().__init__(
          __SCREAMING_SNAKE_CASE   ,				tokenizer_file=__SCREAMING_SNAKE_CASE   ,				do_lower_case=__SCREAMING_SNAKE_CASE   ,				unk_token=__SCREAMING_SNAKE_CASE   ,				sep_token=__SCREAMING_SNAKE_CASE   ,				pad_token=__SCREAMING_SNAKE_CASE   ,				cls_token=__SCREAMING_SNAKE_CASE   ,				mask_token=__SCREAMING_SNAKE_CASE   ,				tokenize_chinese_chars=__SCREAMING_SNAKE_CASE   ,				strip_accents=__SCREAMING_SNAKE_CASE   ,				**__SCREAMING_SNAKE_CASE   ,				)

      __SCREAMING_SNAKE_CASE      = json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
      if (
          normalizer_state.get("""lowercase"""   ,				__SCREAMING_SNAKE_CASE     ) != do_lower_case
          or normalizer_state.get("""strip_accents"""   ,				__SCREAMING_SNAKE_CASE     ) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""   ,				__SCREAMING_SNAKE_CASE     ) != tokenize_chinese_chars
      ):
           __SCREAMING_SNAKE_CASE      = getattr(__SCREAMING_SNAKE_CASE   ,				normalizer_state.pop("""type"""     )     )
           __SCREAMING_SNAKE_CASE      = do_lower_case
           __SCREAMING_SNAKE_CASE      = strip_accents
           __SCREAMING_SNAKE_CASE      = tokenize_chinese_chars
           __SCREAMING_SNAKE_CASE      = normalizer_class(**__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = do_lower_case
 def  UpperCAmelCase__						(						self   :			int   ,				__SCREAMING_SNAKE_CASE   :			List[Any]   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=None     )    ->					Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

      return output
 def  UpperCAmelCase__						(						self   :			int   ,				__SCREAMING_SNAKE_CASE   :			List[int]   ,				__SCREAMING_SNAKE_CASE   :			Optional[List[int]] = None     )    ->					List[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = [self.sep_token_id]
      __SCREAMING_SNAKE_CASE      = [self.cls_token_id]
      if token_ids_a is None:
           return len(cls + token_ids_a + sep     ) * [0]
      return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]


 def  UpperCAmelCase__						(						self   :			Tuple   ,				__SCREAMING_SNAKE_CASE   :			str   ,				__SCREAMING_SNAKE_CASE   :			Optional[str] = None     )    ->					Tuple[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self._tokenizer.model.save(__SCREAMING_SNAKE_CASE   ,				name=__SCREAMING_SNAKE_CASE     )
      return tuple(__SCREAMING_SNAKE_CASE     )









'''simple docstring'''



import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 lowerCAmelCase__       = ["image_processor", "tokenizer"]
 lowerCAmelCase__       = "CLIPImageProcessor"
 lowerCAmelCase__       = ("XLMRobertaTokenizer", "XLMRobertaTokenizerFast")
 def __init__(						self   :			Optional[Any]   ,				__SCREAMING_SNAKE_CASE   :			Any=None   ,				__SCREAMING_SNAKE_CASE   :			str=None   ,				**__SCREAMING_SNAKE_CASE   :			Tuple     )    ->					Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = None
      if "feature_extractor" in kwargs:
           warnings.warn(
               """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
               """ instead."""   ,				__SCREAMING_SNAKE_CASE   ,				)
           __SCREAMING_SNAKE_CASE      = kwargs.pop("""feature_extractor"""     )

      __SCREAMING_SNAKE_CASE      = image_processor if image_processor is not None else feature_extractor
      if image_processor is None:
           raise ValueError("""You need to specify an `image_processor`."""     )
      if tokenizer is None:
           raise ValueError("""You need to specify a `tokenizer`."""     )

      super().__init__(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )
 def __call__(						self   :			Union[str, Any]   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=None   ,				__SCREAMING_SNAKE_CASE   :			Any=None   ,				__SCREAMING_SNAKE_CASE   :			Optional[Any]=None   ,				**__SCREAMING_SNAKE_CASE   :			str     )    ->					int:


      """simple docstring"""




      if text is None and images is None:
           raise ValueError("""You have to specify either text or images. Both cannot be none."""     )

      if text is not None:
           __SCREAMING_SNAKE_CASE      = self.tokenizer(__SCREAMING_SNAKE_CASE   ,				return_tensors=__SCREAMING_SNAKE_CASE   ,				**__SCREAMING_SNAKE_CASE     )

      if images is not None:
           __SCREAMING_SNAKE_CASE      = self.image_processor(__SCREAMING_SNAKE_CASE   ,				return_tensors=__SCREAMING_SNAKE_CASE   ,				**__SCREAMING_SNAKE_CASE     )

      if text is not None and images is not None:
           __SCREAMING_SNAKE_CASE      = image_features.pixel_values
           return encoding
      elif text is not None:
           return encoding
      else:
           return BatchEncoding(data=dict(**__SCREAMING_SNAKE_CASE     )   ,				tensor_type=__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			List[str]   ,				*__SCREAMING_SNAKE_CASE   :			Tuple   ,				**__SCREAMING_SNAKE_CASE   :			List[Any]     )    ->					Optional[int]:


      """simple docstring"""




      return self.tokenizer.batch_decode(*__SCREAMING_SNAKE_CASE   ,				**__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			List[Any]   ,				*__SCREAMING_SNAKE_CASE   :			Union[str, Any]   ,				**__SCREAMING_SNAKE_CASE   :			str     )    ->					Optional[Any]:


      """simple docstring"""




      return self.tokenizer.decode(*__SCREAMING_SNAKE_CASE   ,				**__SCREAMING_SNAKE_CASE     )


 @property
 def  UpperCAmelCase__						(						self   :			List[Any]     )    ->					List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.tokenizer.model_input_names
      __SCREAMING_SNAKE_CASE      = self.image_processor.model_input_names
      return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names     )     )






