'''simple docstring'''




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
  import jax
  import jaxlib

__UpperCAmelCase     							=get_logger()

__UpperCAmelCase     							=None




class     a__      (   TensorFormatter[Mapping, "jax.Array", Mapping] ):






     def __init__( self		:		List[Any]  ,				a		:		int=None  ,				a		:		Dict=None  ,				**a		:		int    ):





         """simple docstring"""



         super().__init__(features=a    )
         import jax
         from jaxlib.xla_client import Device

         if isinstance(a  ,				a    ):
             raise ValueError(
                 f"""Expected {device} to be a `str` not {type(a    )}, as `jaxlib.xla_extension.Device` """
                 '''is not serializable neither with `pickle` nor with `dill`. Instead you can surround '''
                 '''the device with `str()` to get its string identifier that will be internally mapped '''
                 '''to the actual `jaxlib.xla_extension.Device`.'''    )
         __lowerCamelCase		=  device if isinstance(a  ,				a    ) else str(jax.devices()[0]    )
         # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
         # with `pickle` nor with `dill`, so we need to use a global variable instead
         global DEVICE_MAPPING
         if DEVICE_MAPPING is None:
             __lowerCamelCase		=  self._map_devices_to_str()
         if self.device not in list(DEVICE_MAPPING.keys()    ):
             logger.warning(
                 f"""Device with string identifier {self.device} not listed among the available """
                 f"""devices: {list(DEVICE_MAPPING.keys()    )}, so falling back to the default """
                 f"""device: {str(jax.devices()[0]    )}."""    )
             __lowerCamelCase		=  str(jax.devices()[0]    )
         __lowerCamelCase		=  jnp_array_kwargs






     @staticmethod
     def    SCREAMING_SNAKE_CASE__   ( ):





         """simple docstring"""



         import jax

         return {str(a    ): device for device in jax.devices()}






     def    SCREAMING_SNAKE_CASE__   ( self		:		Union[str, Any]  ,				a		:		Dict    ):





         """simple docstring"""



         import jax
         import jax.numpy as jnp

         if isinstance(a  ,				a    ) and column:
             if all(
                 isinstance(a  ,				jax.Array    ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column    ):
                 return jnp.stack(a  ,				axis=0    )
         return column






     def    SCREAMING_SNAKE_CASE__   ( self		:		Tuple  ,				a		:		Dict    ):





         """simple docstring"""



         import jax
         import jax.numpy as jnp

         if isinstance(a  ,				(str, bytes, type(a    ))    ):
             return value
         elif isinstance(a  ,				(np.character, np.ndarray)    ) and np.issubdtype(value.dtype  ,				np.character    ):
             return value.tolist()

         __lowerCamelCase		=  {}

         if isinstance(a  ,				(np.number, np.ndarray)    ) and np.issubdtype(value.dtype  ,				np.integer    ):
             # the default int precision depends on the jax config
             # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
             if jax.config.jax_enable_xaa:
                 __lowerCamelCase		=  {'''dtype''': jnp.intaa}
             else:
                 __lowerCamelCase		=  {'''dtype''': jnp.intaa}
         elif isinstance(a  ,				(np.number, np.ndarray)    ) and np.issubdtype(value.dtype  ,				np.floating    ):
             __lowerCamelCase		=  {'''dtype''': jnp.floataa}
         elif config.PIL_AVAILABLE and "PIL" in sys.modules:
             import PIL.Image

             if isinstance(a  ,				PIL.Image.Image    ):
                 __lowerCamelCase		=  np.asarray(a    )

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
         global DEVICE_MAPPING
         if DEVICE_MAPPING is None:
             __lowerCamelCase		=  self._map_devices_to_str()

         with jax.default_device(DEVICE_MAPPING[self.device]    ):
             # calling jnp.array on a np.ndarray does copy the data
             # see https://github.com/google/jax/issues/4486
             return jnp.array(a  ,				**{**default_dtype, **self.jnp_array_kwargs}    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		Dict  ,				a		:		int    ):





         """simple docstring"""



         import jax

         # support for torch, tf, jax etc.
         if config.TORCH_AVAILABLE and "torch" in sys.modules:
             import torch

             if isinstance(a  ,				torch.Tensor    ):
                 return self._tensorize(data_struct.detach().cpu().numpy()[()]    )
         if hasattr(a  ,				'''__array__'''    ) and not isinstance(a  ,				jax.Array    ):
             __lowerCamelCase		=  data_struct.__array__()
         # support for nested types like struct of list of struct
         if isinstance(a  ,				np.ndarray    ):
             if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
                 return self._consolidate([self.recursive_tensorize(a    ) for substruct in data_struct]    )
         elif isinstance(a  ,				(list, tuple)    ):
             return self._consolidate([self.recursive_tensorize(a    ) for substruct in data_struct]    )
         return self._tensorize(a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		int  ,				a		:		dict    ):





         """simple docstring"""



         return map_nested(self._recursive_tensorize  ,				a  ,				map_list=a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[str]  ,				a		:		pa.Table    ):





         """simple docstring"""



         __lowerCamelCase		=  self.numpy_arrow_extractor().extract_row(a    )
         __lowerCamelCase		=  self.python_features_decoder.decode_row(a    )
         return self.recursive_tensorize(a    )






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[str]  ,				a		:		pa.Table    ):





         """simple docstring"""



         __lowerCamelCase		=  self.numpy_arrow_extractor().extract_column(a    )
         __lowerCamelCase		=  self.python_features_decoder.decode_column(a  ,				pa_table.column_names[0]    )
         __lowerCamelCase		=  self.recursive_tensorize(a    )
         __lowerCamelCase		=  self._consolidate(a    )
         return column






     def    SCREAMING_SNAKE_CASE__   ( self		:		List[str]  ,				a		:		pa.Table    ):





         """simple docstring"""



         __lowerCamelCase		=  self.numpy_arrow_extractor().extract_batch(a    )
         __lowerCamelCase		=  self.python_features_decoder.decode_batch(a    )
         __lowerCamelCase		=  self.recursive_tensorize(a    )
         for column_name in batch:
             __lowerCamelCase		=  self._consolidate(batch[column_name]    )
         return batch




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase           =		'''▁'''

lowerCAmelCase           =		{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase           =		{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

lowerCAmelCase           =		{
    '''google/pegasus-xsum''': 5_1_2,
}


lowerCAmelCase           =		logging.get_logger(__name__)





class       A    (   A_ ):
							UpperCamelCase_   :							Union[str, Any]       =VOCAB_FILES_NAMES

							UpperCamelCase_   :							List[Any]       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							int       =PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							Tuple       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							int       =['''input_ids''', '''attention_mask''']


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase="<pad>"      ,				lowerCAmelCase="</s>"      ,				lowerCAmelCase="<unk>"      ,				lowerCAmelCase="<mask_2>"      ,				lowerCAmelCase="<mask_1>"      ,				lowerCAmelCase=None      ,				lowerCAmelCase=1_0_3      ,				lowerCAmelCase = None      ,				**lowerCAmelCase      ,				):
									__lowercase=			offset
									if additional_special_tokens is not None:
											if not isinstance(lowerCAmelCase      ,				lowerCAmelCase					):
													raise TypeError(
													    f'additional_special_tokens should be of type {type(lowerCAmelCase					)}, but is'
													    f' {type(lowerCAmelCase					)}'					)

											__lowercase=			(
											    ([mask_token_sent] + additional_special_tokens)
											    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
											    else additional_special_tokens
											)
											# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
											additional_special_tokens_extended += [
											    f'<unk_{i}>' for i in range(len(lowerCAmelCase					)      ,				self.offset - 1					)
											]

											if len(set(lowerCAmelCase					)					) != len(lowerCAmelCase					):
													raise ValueError(
													    'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
													    f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'					)
											__lowercase=			additional_special_tokens_extended
									else:
											__lowercase=			[mask_token_sent] if mask_token_sent is not None else []
											additional_special_tokens += [f'<unk_{i}>' for i in range(2      ,				self.offset					)]

									__lowercase=			{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    eos_token=lowerCAmelCase      ,				unk_token=lowerCAmelCase      ,				mask_token=lowerCAmelCase      ,				pad_token=lowerCAmelCase      ,				mask_token_sent=lowerCAmelCase      ,				offset=lowerCAmelCase      ,				additional_special_tokens=lowerCAmelCase      ,				sp_model_kwargs=self.sp_model_kwargs      ,				**lowerCAmelCase      ,				)
									__lowercase=			mask_token_sent
									__lowercase=			vocab_file
									__lowercase=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(lowerCAmelCase					)

									# add special tokens to encoder dict
									__lowercase=			{
									    0: self.pad_token,
									    1: self.eos_token,
									}

									if self.mask_token_sent is not None:
											self.encoder.update(
											    {
											        2: self.mask_token_sent,
											        3: self.mask_token,
											    }					)

									if self.offset > 0:
											# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
											# mask_token_sent is already added to list -> so start at 1
											self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,				self.offset - 1					)}					)

									__lowercase=			{v: k for k, v in self.encoder.items()}


							@property
							def 			_A       (self					):
									return len(self.sp_model					) + self.offset


							def 			_A       (self					):
									__lowercase=			{self.convert_ids_to_tokens(lowerCAmelCase					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab


							def __getstate__(self					):
									__lowercase=			self.__dict__.copy()
									__lowercase=			None
									return state


							def __setstate__(self      ,				lowerCAmelCase					):
									__lowercase=			d

									# for backward compatibility
									if not hasattr(self      ,				'sp_model_kwargs'					):
											__lowercase=			{}

									__lowercase=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(self.vocab_file					)


							def 			_A       (self      ,				lowerCAmelCase					):
									return self.sp_model.encode(lowerCAmelCase      ,				out_type=lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase					):
									if token in self.decoder:
											return self.decoder[token]
									elif token in self.added_tokens_decoder:
											return self.added_tokens_decoder[token]
									__lowercase=			self.sp_model.piece_to_id(lowerCAmelCase					)
									return sp_id + self.offset


							def 			_A       (self      ,				lowerCAmelCase					):
									if index in self.encoder:
											return self.encoder[index]
									elif index in self.added_tokens_encoder:
											return self.added_tokens_encoder[index]
									else:
											__lowercase=			self.sp_model.IdToPiece(index - self.offset					)
									return token


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			[]
									__lowercase=			''
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													out_string += self.sp_model.decode(lowerCAmelCase					) + token
													__lowercase=			[]
											else:
													current_sub_tokens.append(lowerCAmelCase					)
									out_string += self.sp_model.decode(lowerCAmelCase					)
									return out_string.strip()


							def 			_A       (self      ,				lowerCAmelCase=False					):
									return 1


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			set(self.all_special_ids					)  # call it once instead of inside list comp
									all_special_ids.remove(self.unk_token_id					)  # <unk> is only sometimes special

									return [1 if x in all_special_ids else 0 for x in seq]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None      ,				lowerCAmelCase = False					):
									if already_has_special_tokens:
											return self._special_token_mask(lowerCAmelCase					)
									elif token_ids_a is None:
											return self._special_token_mask(lowerCAmelCase					) + [1]
									else:
											return self._special_token_mask(token_ids_a + token_ids_a					) + [1]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None					):
									if token_ids_a is None:
											return token_ids_a + [self.eos_token_id]
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return token_ids_a + token_ids_a + [self.eos_token_id]







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									if not os.path.isdir(lowerCAmelCase					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
									__lowercase=			os.path.join(
									    lowerCAmelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase					) and os.path.isfile(self.vocab_file					):
											copyfile(self.vocab_file      ,				lowerCAmelCase					)
									elif not os.path.isfile(self.vocab_file					):
											with open(lowerCAmelCase      ,				'wb'					) as fi:
													__lowercase=			self.sp_model.serialized_model_proto()
													fi.write(lowerCAmelCase					)

									return (out_vocab_file,)






