

'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


__lowerCamelCase										=			logging.get_logger(__name__)

__lowerCamelCase										=			{"tokenizer_file": "tokenizer.json"}

__lowerCamelCase										=			{
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}


class     A__					(					SCREAMING_SNAKE_CASE__	):
    lowercase		    =						VOCAB_FILES_NAMES
    lowercase		    =						PRETRAINED_VOCAB_FILES_MAP
    lowercase		    =						["input_ids", "attention_mask"]
    lowercase		    =						None



    def __init__(    self						, UpperCamelCase__=None						, UpperCamelCase__=None						, UpperCamelCase__=None						, UpperCamelCase__="<unk>"						, UpperCamelCase__="<s>"						, UpperCamelCase__="</s>"						, UpperCamelCase__="<pad>"						, UpperCamelCase__=False						, UpperCamelCase__=False						, **UpperCamelCase__						, )  ->					Optional[Any]:





           '''simple docstring'''
           super().__init__(
               UpperCamelCase__						, UpperCamelCase__						, tokenizer_file=UpperCamelCase__						, unk_token=UpperCamelCase__						, bos_token=UpperCamelCase__						, eos_token=UpperCamelCase__						, pad_token=UpperCamelCase__						, add_prefix_space=UpperCamelCase__						, clean_up_tokenization_spaces=UpperCamelCase__						, **UpperCamelCase__						, )
           A_		    =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
           if pre_tok_state.get("""add_prefix_space"""						, UpperCamelCase__				) != add_prefix_space:
                  A_		    =					getattr(UpperCamelCase__						, pre_tok_state.pop("""type"""				)				)
                  A_		    =					add_prefix_space
                  A_		    =					pre_tok_class(**UpperCamelCase__				)

           A_		    =					add_prefix_space



    def     snake_case_	(    self						, *UpperCamelCase__						, **UpperCamelCase__				)  ->					BatchEncoding:





           '''simple docstring'''
           A_		    =					kwargs.get("""is_split_into_words"""						, UpperCamelCase__				)
           if not (self.add_prefix_space or not is_split_into_words):
                  raise Exception(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                      """ pretokenized inputs."""				)

           return super()._batch_encode_plus(*UpperCamelCase__						, **UpperCamelCase__				)



    def     snake_case_	(    self						, *UpperCamelCase__						, **UpperCamelCase__				)  ->					BatchEncoding:





           '''simple docstring'''
           A_		    =					kwargs.get("""is_split_into_words"""						, UpperCamelCase__				)

           if not (self.add_prefix_space or not is_split_into_words):
                  raise Exception(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                      """ pretokenized inputs."""				)

           return super()._encode_plus(*UpperCamelCase__						, **UpperCamelCase__				)



    def     snake_case_	(    self						, UpperCamelCase__						, UpperCamelCase__ = None				)  ->					Tuple[str]:





           '''simple docstring'''
           A_		    =					self._tokenizer.model.save(UpperCamelCase__						, name=UpperCamelCase__				)
           return tuple(UpperCamelCase__				)







    def     snake_case_	(    self						, UpperCamelCase__				)  ->					List[int]:





           '''simple docstring'''
           A_		    =					[]
           for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(UpperCamelCase__						, add_special_tokens=UpperCamelCase__				) + [self.eos_token_id]				)

           if len(UpperCamelCase__				) > self.model_max_length:
                  A_		    =					input_ids[-self.model_max_length :]
           return input_ids

from ...processing_utils import ProcessorMixin


class      lowerCamelCase    (SCREAMING_SNAKE_CASE__    ):
  """simple docstring"""







  lowerCamelCase__   =  ['''image_processor''', '''feature_extractor''']
  lowerCamelCase__   =  '''TvltImageProcessor'''
  lowerCamelCase__   =  '''TvltFeatureExtractor'''







  def __init__(							self       :				List[str]     ,   __magic_name__       :				Any     ,   __magic_name__       :				Any      )     ->		int:
        super().__init__(image_processor=__magic_name__     ,   feature_extractor=__magic_name__      )

        SCREAMING_SNAKE_CASE_      =     image_processor
        SCREAMING_SNAKE_CASE_      =     feature_extractor







  def __call__(							self       :				List[str]     ,   __magic_name__       :				Optional[Any]=None     ,   __magic_name__       :				Union[str, Any]=None     ,   __magic_name__       :				int=None     ,   __magic_name__       :				str=None     ,   __magic_name__       :				Any=False     ,   __magic_name__       :				int=False     ,   *__magic_name__       :				int     ,   **__magic_name__       :				Any     ,   )     ->		List[Any]:

        if images is None and audio is None:
              raise ValueError("You need to specify either an `images` or `audio` input to process."      )

        SCREAMING_SNAKE_CASE_      =     None
        if images is not None:
              SCREAMING_SNAKE_CASE_      =     self.image_processor(__magic_name__     ,   mask_pixel=__magic_name__     ,   *__magic_name__     ,   **__magic_name__      )
        if images_mixed is not None:
              SCREAMING_SNAKE_CASE_      =     self.image_processor(__magic_name__     ,   is_mixed=__magic_name__     ,   *__magic_name__     ,   **__magic_name__      )
        if audio is not None:
              SCREAMING_SNAKE_CASE_      =     self.feature_extractor(
                  __magic_name__     ,   *__magic_name__     ,   sampling_rate=__magic_name__     ,   mask_audio=__magic_name__     ,   **__magic_name__      )

        SCREAMING_SNAKE_CASE_      =     {}
        if audio is not None:
              output_dict.update(__magic_name__      )
        if images is not None:
              output_dict.update(__magic_name__      )
        if images_mixed_dict is not None:
              output_dict.update(__magic_name__      )
        return output_dict







  @property
  def       __A   (							self       :				Optional[Any]      )     ->		int:
        SCREAMING_SNAKE_CASE_      =     self.image_processor.model_input_names
        SCREAMING_SNAKE_CASE_      =     self.feature_extractor.model_input_names
        return list(dict.fromkeys(image_processor_input_names + feature_extractor_input_names      )      )


