




"""simple docstring"""

# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

_lowerCamelCase     :	Optional[Any]         =				get_logger()

_lowerCamelCase     :	Optional[dict]         =				None

class 				lowercase					(			TensorFormatter[Mapping, """jax.Array""", Mapping]):







				def __init__(							self      :    int					,   _lowerCamelCase      :    Optional[int]=None					,   _lowerCamelCase      :    Union[str, Any]=None					,   **_lowerCamelCase      :    Union[str, Any]				):



					"""simple docstring"""

					super().__init__(features=lowerCAmelCase__				)
					import jax
					from jaxlib.xla_client import Device

					if isinstance(lowerCAmelCase__					,   lowerCAmelCase__				):
						raise ValueError(
						    F"""Expected {device} to be a `str` not {type(lowerCAmelCase__				)}, as `jaxlib.xla_extension.Device` """
						    '''is not serializable neither with `pickle` nor with `dill`. Instead you can surround '''
						    '''the device with `str()` to get its string identifier that will be internally mapped '''
						    '''to the actual `jaxlib.xla_extension.Device`.'''				)
					A_     : List[Any]		      =						device if isinstance(lowerCAmelCase__					,   lowerCAmelCase__				) else str(jax.devices()[0]				)
					# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
					# with `pickle` nor with `dill`, so we need to use a global variable instead
					global DEVICE_MAPPING
					if DEVICE_MAPPING is None:
						A_     : int		      =						self._map_devices_to_str()
					if self.device not in list(DEVICE_MAPPING.keys()				):
						logger.warning(
						    F"""Device with string identifier {self.device} not listed among the available """
						    F"""devices: {list(DEVICE_MAPPING.keys()				)}, so falling back to the default """
						    F"""device: {str(jax.devices()[0]				)}."""				)
						A_     : int		      =						str(jax.devices()[0]				)
					A_     : Union[str, Any]		      =						jnp_array_kwargs







				@staticmethod
				def     a_     (							):



					"""simple docstring"""

					import jax

					return {str(lowerCAmelCase__				): device for device in jax.devices()}







				def     a_     (							self      :    Dict					,   _lowerCamelCase      :    List[Any]				):



					"""simple docstring"""

					import jax
					import jax.numpy as jnp

					if isinstance(lowerCAmelCase__					,   lowerCAmelCase__				) and column:
						if all(
						    isinstance(lowerCAmelCase__					,   jax.Array				) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column				):
							return jnp.stack(lowerCAmelCase__					,   axis=0				)
					return column







				def     a_     (							self      :    Optional[Any]					,   _lowerCamelCase      :    List[Any]				):



					"""simple docstring"""

					import jax
					import jax.numpy as jnp

					if isinstance(lowerCAmelCase__					,   (str, bytes, type(lowerCAmelCase__				))				):
						return value
					elif isinstance(lowerCAmelCase__					,   (np.character, np.ndarray)				) and np.issubdtype(value.dtype					,   np.character				):
						return value.tolist()

					A_     : Optional[Any]		      =						{}

					if isinstance(lowerCAmelCase__					,   (np.number, np.ndarray)				) and np.issubdtype(value.dtype					,   np.integer				):
						# the default int precision depends on the jax config
						# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
						if jax.config.jax_enable_xaa:
							A_     : Optional[Any]		      =						{"dtype": jnp.intaa}
						else:
							A_     : Dict		      =						{"dtype": jnp.intaa}
					elif isinstance(lowerCAmelCase__					,   (np.number, np.ndarray)				) and np.issubdtype(value.dtype					,   np.floating				):
						A_     : Tuple		      =						{"dtype": jnp.floataa}
					elif config.PIL_AVAILABLE and "PIL" in sys.modules:
						import PIL.Image

						if isinstance(lowerCAmelCase__					,   PIL.Image.Image				):
							A_     : Tuple		      =						np.asarray(lowerCAmelCase__				)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
					global DEVICE_MAPPING
					if DEVICE_MAPPING is None:
						A_     : List[str]		      =						self._map_devices_to_str()

					with jax.default_device(DEVICE_MAPPING[self.device]				):
						# calling jnp.array on a np.ndarray does copy the data
						# see https://github.com/google/jax/issues/4486
						return jnp.array(lowerCAmelCase__					,   **{**default_dtype, **self.jnp_array_kwargs}				)







				def     a_     (							self      :    Tuple					,   _lowerCamelCase      :    str				):



					"""simple docstring"""

					import jax

					# support for torch, tf, jax etc.
					if config.TORCH_AVAILABLE and "torch" in sys.modules:
						import torch

						if isinstance(lowerCAmelCase__					,   torch.Tensor				):
							return self._tensorize(data_struct.detach().cpu().numpy()[()]				)
					if hasattr(lowerCAmelCase__					,   '''__array__'''				) and not isinstance(lowerCAmelCase__					,   jax.Array				):
						A_     : List[Any]		      =						data_struct.__array__()
					# support for nested types like struct of list of struct
					if isinstance(lowerCAmelCase__					,   np.ndarray				):
						if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
							return self._consolidate([self.recursive_tensorize(lowerCAmelCase__				) for substruct in data_struct]				)
					elif isinstance(lowerCAmelCase__					,   (list, tuple)				):
						return self._consolidate([self.recursive_tensorize(lowerCAmelCase__				) for substruct in data_struct]				)
					return self._tensorize(lowerCAmelCase__				)







				def     a_     (							self      :    Tuple					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					return map_nested(self._recursive_tensorize					,   lowerCAmelCase__					,   map_list=lowerCAmelCase__				)







				def     a_     (							self      :    Union[str, Any]					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					A_     : Dict		      =						self.numpy_arrow_extractor().extract_row(lowerCAmelCase__				)
					A_     : List[Any]		      =						self.python_features_decoder.decode_row(lowerCAmelCase__				)
					return self.recursive_tensorize(lowerCAmelCase__				)







				def     a_     (							self      :    Any					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					A_     : Optional[int]		      =						self.numpy_arrow_extractor().extract_column(lowerCAmelCase__				)
					A_     : str		      =						self.python_features_decoder.decode_column(lowerCAmelCase__					,   pa_table.column_names[0]				)
					A_     : int		      =						self.recursive_tensorize(lowerCAmelCase__				)
					A_     : Tuple		      =						self._consolidate(lowerCAmelCase__				)
					return column







				def     a_     (							self      :    Optional[int]					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					A_     : Optional[int]		      =						self.numpy_arrow_extractor().extract_batch(lowerCAmelCase__				)
					A_     : Any		      =						self.python_features_decoder.decode_batch(lowerCAmelCase__				)
					A_     : List[str]		      =						self.recursive_tensorize(lowerCAmelCase__				)
					for column_name in batch:
						A_     : int		      =						self._consolidate(batch[column_name]				)
					return batch












from queue import Queue
from typing import TYPE_CHECKING, Optional


if TYPE_CHECKING:
     from ..models.auto import AutoTokenizer







class 				__lowerCAmelCase       :







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        raise NotImplementedError()







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        raise NotImplementedError()







class 				__lowerCAmelCase       (    UpperCamelCase__):







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = False		,      **lowerCAmelCase__							)   ->						int:



        '''simple docstring'''




        a__	:					str										=tokenizer
        a__	:					List[str]										=skip_prompt
        a__	:					List[Any]										=decode_kwargs

        # variables used in the streaming process
        a__	:					Dict										=[]
        a__	:					int										=0
        a__	:					str										=True







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Union[str, Any]:



        '''simple docstring'''




        if len(value.shape							) > 1 and value.shape[0] > 1:
          raise ValueError("TextStreamer only supports batch size 1"							)
        elif len(value.shape							) > 1:
          a__	:					Any										=value[0]

        if self.skip_prompt and self.next_tokens_are_prompt:
          a__	:					Dict										=False
          return

        # Add the new token to the cache and decodes the entire thing.
        self.token_cache.extend(value.tolist()							)
        a__	:					Union[str, Any]										=self.tokenizer.decode(self.token_cache		,      **self.decode_kwargs							)

        # After the symbol for a new line, we flush the cache.
        if text.endswith("\n"							):
          a__	:					List[Any]										=text[self.print_len :]
          a__	:					List[str]										=[]
          a__	:					Optional[int]										=0
        # If the last token is a CJK character, we print the characters.
        elif len(lowerCAmelCase__							) > 0 and self._is_chinese_char(ord(text[-1]							)							):
          a__	:					List[str]										=text[self.print_len :]
          self.print_len += len(lowerCAmelCase__							)
        # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words,
        # which may change with the subsequent token -- there are probably smarter ways to do this!)
        else:
          a__	:					str										=text[self.print_len : text.rfind(" "							) + 1]
          self.print_len += len(lowerCAmelCase__							)

        self.on_finalized_text(lowerCAmelCase__							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        if len(self.token_cache							) > 0:
          a__	:					Union[str, Any]										=self.tokenizer.decode(self.token_cache		,      **self.decode_kwargs							)
          a__	:					List[Any]										=text[self.print_len :]
          a__	:					List[str]										=[]
          a__	:					Optional[int]										=0
        else:
          a__	:					Union[str, Any]										=""

        a__	:					Any										=True
        self.on_finalized_text(lowerCAmelCase__		,      stream_end=lowerCAmelCase__							)







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = False							)   ->						Optional[Any]:



        '''simple docstring'''




        print(lowerCAmelCase__		,      flush=lowerCAmelCase__		,      end="" if not stream_end else None							)







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						str:



        '''simple docstring'''




        if (
            (cp >= 0X4E_00 and cp <= 0X9F_FF)
            or (cp >= 0X34_00 and cp <= 0X4D_BF)  #
            or (cp >= 0X2_00_00 and cp <= 0X2_A6_DF)  #
            or (cp >= 0X2_A7_00 and cp <= 0X2_B7_3F)  #
            or (cp >= 0X2_B7_40 and cp <= 0X2_B8_1F)  #
            or (cp >= 0X2_B8_20 and cp <= 0X2_CE_AF)  #
            or (cp >= 0XF9_00 and cp <= 0XFA_FF)
            or (cp >= 0X2_F8_00 and cp <= 0X2_FA_1F)  #
        ):  #
          return True

        return False







class 				__lowerCAmelCase       (    UpperCamelCase__):







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = False		,      lowerCAmelCase__ = None		,      **lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        super().__init__(lowerCAmelCase__		,      lowerCAmelCase__		,      **lowerCAmelCase__							)
        a__	:					str										=Queue()
        a__	:					Optional[Any]										=None
        a__	:					Any										=timeout







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = False							)   ->						List[str]:



        '''simple docstring'''




        self.text_queue.put(lowerCAmelCase__		,      timeout=self.timeout							)
        if stream_end:
          self.text_queue.put(self.stop_signal		,      timeout=self.timeout							)







      def __iter__(    self							)   ->						Dict:



        '''simple docstring'''




        return self







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        a__	:					int										=self.text_queue.get(timeout=self.timeout							)
        if value == self.stop_signal:
          raise StopIteration()
        else:
          return value






