



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE_:Optional[int]             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Tuple             =						{"""tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_:Dict             =						{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[Any]					=      VOCAB_FILES_NAMES
    __lowerCamelCase			: Optional[Any]					=      PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase			: Optional[int]					=      ["input_ids", "attention_mask"]
    __lowerCamelCase			: List[str]					=      None






    def __init__(       self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__="<unk>",		lowerCamelCase__="<s>",		lowerCamelCase__="</s>",		lowerCamelCase__="<pad>",		lowerCamelCase__=False,		lowerCamelCase__=False,		**lowerCamelCase__,		):
        super().__init__(
            lowerCamelCase__,		lowerCamelCase__,		tokenizer_file=lowerCamelCase__,		unk_token=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		add_prefix_space=lowerCamelCase__,		clean_up_tokenization_spaces=lowerCamelCase__,		**lowerCamelCase__,		)
        A :       str          =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
        if pre_tok_state.get("""add_prefix_space""",		lowerCamelCase__		) != add_prefix_space:
            A :       Optional[int]          =   getattr(lowerCamelCase__,		pre_tok_state.pop("""type"""		)		)
            A :       str          =   add_prefix_space
            A :       Any          =   pre_tok_class(**lowerCamelCase__		)

        A :       Union[str, Any]          =   add_prefix_space






    def 			_lowerCAmelCase					(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        A :       str          =   kwargs.get("""is_split_into_words""",		lowerCamelCase__		)
        if not (self.add_prefix_space or not is_split_into_words):
            raise Exception(
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                """ pretokenized inputs."""		)

        return super()._batch_encode_plus(*lowerCamelCase__,		**lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        A :       Union[str, Any]          =   kwargs.get("""is_split_into_words""",		lowerCamelCase__		)

        if not (self.add_prefix_space or not is_split_into_words):
            raise Exception(
                f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                """ pretokenized inputs."""		)

        return super()._encode_plus(*lowerCamelCase__,		**lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       str          =   self._tokenizer.model.save(lowerCamelCase__,		name=lowerCamelCase__		)
        return tuple(lowerCamelCase__		)






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__		):
        A :       Any          =   []
        for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		) + [self.eos_token_id]		)

        if len(lowerCamelCase__		) > self.model_max_length:
            A :       Dict          =   input_ids[-self.model_max_length :]
        return input_ids







from ..utils import DummyObject, requires_backends






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Dict					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Any					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






class SCREAMING_SNAKE_CASE__     (		metaclass=SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[int]					=      ["torch", "transformers", "onnx"]






    def __init__(       self,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(self,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)






    @classmethod
    def 			_lowerCAmelCase					(       cls,		*lowerCamelCase__,		**lowerCamelCase__		):
        requires_backends(cls,		["""torch""", """transformers""", """onnx"""]		)


