'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_:	List[str]    =get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class 			__A (					UpperCamelCase__						,      unittest.TestCase      ):
    a__			:		str			  =    ReformerTokenizer
    a__			:		Optional[Any]			  =    ReformerTokenizerFast
    a__			:		Any			  =    True
    a__			:		Union[str, Any]			  =    False
    a__			:		int			  =    True



    def    _lowercase		(self       :       List[str]					):
         super().setUp()

         UpperCAmelCase_			       =  ReformerTokenizer(__a  ,     keep_accents=__a					)
         tokenizer.save_pretrained(self.tmpdirname					)



    def    _lowercase		(self       :       int					):
         UpperCAmelCase_			       =  "<s>"
         UpperCAmelCase_			       =  1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a					)  ,     __a					)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a					)  ,     __a					)



    def    _lowercase		(self       :       Optional[Any]					):
         UpperCAmelCase_			       =  list(self.get_tokenizer().get_vocab().keys()					)

         self.assertEqual(vocab_keys[0]  ,     "<unk>"					)
         self.assertEqual(vocab_keys[1]  ,     "<s>"					)
         self.assertEqual(vocab_keys[-1]  ,     "j"					)
         self.assertEqual(len(__a					)  ,     1000					)



    def    _lowercase		(self       :       str					):
         self.assertEqual(self.get_tokenizer().vocab_size  ,     1000					)



    def    _lowercase		(self       :       Dict					):
         if not self.test_rust_tokenizer:
              return

         UpperCAmelCase_			       =  self.get_tokenizer()
         UpperCAmelCase_			       =  self.get_rust_tokenizer()

         UpperCAmelCase_			       =  "I was born in 92000, and this is falsé."

         UpperCAmelCase_			       =  tokenizer.tokenize(__a					)
         UpperCAmelCase_			       =  rust_tokenizer.tokenize(__a					)
         self.assertListEqual(__a  ,     __a					)

         UpperCAmelCase_			       =  tokenizer.encode(__a  ,     add_special_tokens=__a					)
         UpperCAmelCase_			       =  rust_tokenizer.encode(__a  ,     add_special_tokens=__a					)
         self.assertListEqual(__a  ,     __a					)

         UpperCAmelCase_			       =  self.get_rust_tokenizer()
         UpperCAmelCase_			       =  tokenizer.encode(__a					)
         UpperCAmelCase_			       =  rust_tokenizer.encode(__a					)
         self.assertListEqual(__a  ,     __a					)



    def    _lowercase		(self       :       Union[str, Any]  ,     __a       :       List[str]=15					):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
                   UpperCAmelCase_			       =  self.rust_tokenizer_class.from_pretrained(__a  ,     **__a					)

                   # Simple input
                   UpperCAmelCase_			       =  "This is a simple input"
                   UpperCAmelCase_			       =  ["This is a simple input 1", "This is a simple input 2"]
                   UpperCAmelCase_			       =  ("This is a simple input", "This is a pair")
                   UpperCAmelCase_			       =  [
                       ("This is a simple input 1", "This is a simple input 2"),
                       ("This is a simple pair 1", "This is a simple pair 2"),
                   ]

                   # Simple input tests
                   self.assertRaises(__a  ,     tokenizer_r.encode  ,     __a  ,     max_length=__a  ,     padding="max_length"					)

                   # Simple input
                   self.assertRaises(__a  ,     tokenizer_r.encode_plus  ,     __a  ,     max_length=__a  ,     padding="max_length"					)

                   # Simple input
                   self.assertRaises(
                       __a  ,     tokenizer_r.batch_encode_plus  ,     __a  ,     max_length=__a  ,     padding="max_length"  ,     )

                   # Pair input
                   self.assertRaises(__a  ,     tokenizer_r.encode  ,     __a  ,     max_length=__a  ,     padding="max_length"					)

                   # Pair input
                   self.assertRaises(__a  ,     tokenizer_r.encode_plus  ,     __a  ,     max_length=__a  ,     padding="max_length"					)

                   # Pair input
                   self.assertRaises(
                       __a  ,     tokenizer_r.batch_encode_plus  ,     __a  ,     max_length=__a  ,     padding="max_length"  ,     )



    def    _lowercase		(self       :       Optional[Any]					):
         pass



    def    _lowercase		(self       :       Optional[int]					):
         UpperCAmelCase_			       =  ReformerTokenizer(__a  ,     keep_accents=__a					)

         UpperCAmelCase_			       =  tokenizer.tokenize("This is a test"					)
         self.assertListEqual(__a  ,     ["▁This", "▁is", "▁a", "▁t", "est"]					)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(__a					)  ,     [285, 46, 10, 170, 382]  ,     )

         UpperCAmelCase_			       =  tokenizer.tokenize("I was born in 92000, and this is falsé."					)
         self.assertListEqual(
             __a  ,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]  ,     )
         UpperCAmelCase_			       =  tokenizer.convert_tokens_to_ids(__a					)
         self.assertListEqual(
             __a  ,     [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]  ,     )

         UpperCAmelCase_			       =  tokenizer.convert_ids_to_tokens(__a					)
         self.assertListEqual(
             __a  ,     [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]  ,     )



    @cached_property
    def    _lowercase		(self       :       Optional[int]					):
         return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"					)



    @slow
    def    _lowercase		(self       :       Union[str, Any]					):
         UpperCAmelCase_			       =  "Hello World!"
         UpperCAmelCase_			       =  [126, 32, 262, 152, 38, 72, 287]

         self.assertListEqual(__a  ,     self.big_tokenizer.encode(__a					)					)



    @slow
    def    _lowercase		(self       :       Optional[Any]					):
         UpperCAmelCase_			       =  (
             "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
             " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
         )
         UpperCAmelCase_			       =  [
             108,
             265,
             24,
             111,
             4,
             258,
             156,
             35,
             28,
             275,
             3,
             259,
             297,
             260,
             84,
             4,
             35,
             110,
             44,
             8,
             259,
             91,
             268,
             21,
             11,
             209,
             274,
             109,
             266,
             277,
             117,
             86,
             93,
             315,
             258,
             278,
             258,
             277,
             258,
             0,
             258,
             288,
             258,
             319,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             0,
             258,
             287,
             258,
             315,
             258,
             289,
             258,
             278,
             99,
             269,
             266,
             262,
             8,
             259,
             241,
             4,
             217,
             230,
             268,
             266,
             55,
             168,
             106,
             75,
             193,
             266,
             223,
             27,
             49,
             26,
             282,
             25,
             264,
             299,
             19,
             26,
             0,
             258,
             277,
             117,
             86,
             93,
             176,
             183,
             270,
             11,
             262,
             42,
             61,
             265,
         ]

         self.assertListEqual(__a  ,     self.big_tokenizer.encode(__a					)					)



    @require_torch
    @slow
    def    _lowercase		(self       :       Tuple					):
         import torch

         from transformers import ReformerConfig, ReformerModel

         # Build sequence
         UpperCAmelCase_			       =  list(self.big_tokenizer.get_vocab().keys()					)[:10]
         UpperCAmelCase_			       =  " ".join(__a					)
         UpperCAmelCase_			       =  self.big_tokenizer.encode_plus(__a  ,     return_tensors="pt"					)
         UpperCAmelCase_			       =  self.big_tokenizer.batch_encode_plus([sequence, sequence]  ,     return_tensors="pt"					)

         UpperCAmelCase_			       =  ReformerConfig()
         # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
         UpperCAmelCase_			       =  encoded_sequence["input_ids"].shape
         UpperCAmelCase_			       =  ReformerModel(__a					)

         # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
              model(**__a					)
              model(**__a					)






    @slow
    def    _lowercase		(self       :       Dict					):
         # fmt: off
         UpperCAmelCase_			       =  {"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # This tokenizer does not know some characters like ")".
         # That is the reason why we use very simple texts here.
         # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
         UpperCAmelCase_			       =  [
             "This is a very simple sentence.",
             "The quick brown fox jumps over the lazy dog.",
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=__a  ,     model_name="google/reformer-crime-and-punishment"  ,     revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"  ,     padding=__a  ,     sequences=__a  ,     )












"""simple docstring"""
import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


lowerCAmelCase__    						=			get_logger(__name__)


lowerCAmelCase__    						=			r'\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n'


class _lowerCAmelCase						:







						@add_start_docstrings(lowerCAmelCase_							)
						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										raise NotImplementedError(
										    F"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)







class _lowerCAmelCase						:







						@add_start_docstrings(lowerCAmelCase_							)
						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										raise NotImplementedError(
										    F"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						@add_start_docstrings(lowerCAmelCase_							)
						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				**lowerCAmelCase_							)		->							jnp.ndarray:
										for processor in self:
														_SCREAMING_SNAKE_CASE   :		int       		=    inspect.signature(processor.__call__							).parameters
														if len(lowerCAmelCase_							) > 3:
																		if not all(arg in kwargs for arg in list(function_args.keys()							)[2:]							):
																						raise ValueError(
																						    F"""Make sure that all the required parameters: {list(function_args.keys()							)} for """
																						    F"""{processor.__class__} are passed to the logits processor."""							)
																		_SCREAMING_SNAKE_CASE   :		Any       		=    processor(lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				**lowerCAmelCase_							)
														else:
																		_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    processor(lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)
										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_							)		->							Any:
										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or not (temperature > 0):
														raise ValueError(F"""`temperature` has to be a strictly positive float, but is {temperature}"""							)

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    temperature







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    scores / self.temperature
										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = -float('Inf'							)   ,				lowerCAmelCase_ = 1							)		->							int:
										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or (top_p < 0 or top_p > 1.0):
														raise ValueError(F"""`top_p` has to be a float > 0 and < 1, but is {top_p}"""							)
										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or (min_tokens_to_keep < 1):
														raise ValueError(F"""`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"""							)

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    top_p
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    filter_value
										_SCREAMING_SNAKE_CASE   :		Any       		=    min_tokens_to_keep







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE   :		Dict       		=    lax.top_k(lowerCAmelCase_   ,				scores.shape[-1]							)

										_SCREAMING_SNAKE_CASE   :		Dict       		=    jnp.full_like(lowerCAmelCase_   ,				self.filter_value							)
										_SCREAMING_SNAKE_CASE   :		int       		=    jax.nn.softmax(lowerCAmelCase_   ,				axis=-1							).cumsum(axis=-1							)
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    cumulative_probs < self.top_p

										# include the token that is higher than top_p as well
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    jnp.roll(lowerCAmelCase_   ,				1							)
										score_mask |= score_mask.at[:, 0].set(lowerCAmelCase_							)

										# min tokens to keep
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    score_mask.at[:, : self.min_tokens_to_keep].set(lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    jnp.where(lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Dict       		=    jax.lax.sort_key_val(lowerCAmelCase_   ,				lowerCAmelCase_							)[-1]

										return next_scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_ = -float('Inf'							)   ,				lowerCAmelCase_ = 1							)		->							Tuple:
										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or top_k <= 0:
														raise ValueError(F"""`top_k` has to be a strictly positive integer, but is {top_k}"""							)

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    max(lowerCAmelCase_   ,				lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    filter_value







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE   :		int       		=    scores.shape
										_SCREAMING_SNAKE_CASE   :		int       		=    jnp.full(batch_size * vocab_size   ,				self.filter_value							)

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    min(self.top_k   ,				scores.shape[-1]							)  # Safety check
										_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE   :		List[Any]       		=    lax.top_k(lowerCAmelCase_   ,				lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    jnp.broadcast_to((jnp.arange(lowerCAmelCase_							) * vocab_size)[:, None]   ,				(batch_size, topk)							).flatten()
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    topk_scores.flatten()
										_SCREAMING_SNAKE_CASE   :		Dict       		=    topk_indices.flatten() + shift

										_SCREAMING_SNAKE_CASE   :		List[str]       		=    next_scores_flat.at[topk_indices_flat].set(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		int       		=    next_scores_flat.reshape(lowerCAmelCase_   ,				lowerCAmelCase_							)
										return next_scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_							)		->							Dict:
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    bos_token_id







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    jnp.full(scores.shape   ,				-float('inf'							)							)

										_SCREAMING_SNAKE_CASE   :		int       		=    1 - jnp.bool_(cur_len - 1							)

										_SCREAMING_SNAKE_CASE   :		List[str]       		=    jnp.where(lowerCAmelCase_   ,				new_scores.at[:, self.bos_token_id].set(0							)   ,				lowerCAmelCase_							)

										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							List[Any]:
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    max_length
										_SCREAMING_SNAKE_CASE   :		Any       		=    eos_token_id







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    jnp.full(scores.shape   ,				-float('inf'							)							)

										_SCREAMING_SNAKE_CASE   :		Dict       		=    1 - jnp.bool_(cur_len - self.max_length + 1							)

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    jnp.where(lowerCAmelCase_   ,				new_scores.at[:, self.eos_token_id].set(0							)   ,				lowerCAmelCase_							)

										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							Optional[int]:
										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or min_length < 0:
														raise ValueError(F"""`min_length` has to be a positive integer, but is {min_length}"""							)

										if not isinstance(lowerCAmelCase_   ,				lowerCAmelCase_							) or eos_token_id < 0:
														raise ValueError(F"""`eos_token_id` has to be a positive integer, but is {eos_token_id}"""							)

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    min_length
										_SCREAMING_SNAKE_CASE   :		str       		=    eos_token_id







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										# create boolean flag to decide if min length penalty should be applied
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    1 - jnp.clip(cur_len - self.min_length   ,				0   ,				1							)

										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    jnp.where(lowerCAmelCase_   ,				scores.at[:, self.eos_token_id].set(-float('inf'							)							)   ,				lowerCAmelCase_							)

										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							List[str]:
										_SCREAMING_SNAKE_CASE   :		List[str]       		=    list(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    begin_index







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							str:
										_SCREAMING_SNAKE_CASE   :		Any       		=    1 - jnp.bool_(cur_len - self.begin_index							)

										_SCREAMING_SNAKE_CASE   :		List[Any]       		=    jnp.where(lowerCAmelCase_   ,				scores.at[:, self.begin_suppress_tokens].set(-float('inf'							)							)   ,				lowerCAmelCase_							)

										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_							)		->							Optional[Any]:
										_SCREAMING_SNAKE_CASE   :		str       		=    list(lowerCAmelCase_							)







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    scores.at[..., self.suppress_tokens].set(-float('inf'							)							)

										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_							)		->							Any:
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    dict(lowerCAmelCase_							)
										# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
										# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
										# Indexes without forced tokens will have a negative value.
										_SCREAMING_SNAKE_CASE   :		Dict       		=    jnp.ones((max(force_token_map.keys()							) + 1)   ,				dtype=jnp.intaa							) * -1
										for index, token in force_token_map.items():
														if token is not None:
																		_SCREAMING_SNAKE_CASE   :		Dict       		=    force_token_array.at[index].set(lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		str       		=    jnp.intaa(lowerCAmelCase_							)







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							jnp.ndarray:
										def _force_token(lowerCAmelCase_							):
														_SCREAMING_SNAKE_CASE   :		str       		=    scores.shape[0]
														_SCREAMING_SNAKE_CASE   :		List[Any]       		=    self.force_token_array[generation_idx]

														_SCREAMING_SNAKE_CASE   :		Dict       		=    jnp.ones_like(lowerCAmelCase_   ,				dtype=scores.dtype							) * -float('inf'							)
														_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    jnp.zeros((batch_size, 1)   ,				dtype=scores.dtype							)
														_SCREAMING_SNAKE_CASE   :		str       		=    lax.dynamic_update_slice(lowerCAmelCase_   ,				lowerCAmelCase_   ,				(0, current_token)							)
														return new_scores

										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    lax.cond(
										    cur_len >= self.force_token_array.shape[0]   ,				lambda: scores   ,				lambda: lax.cond(
										        self.force_token_array[cur_len] >= 0   ,				lambda: _force_token(lowerCAmelCase_							)   ,				lambda: scores   ,				)   ,				)
										return scores







class _lowerCAmelCase						(    __UpperCAmelCase       ):







						def __init__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							str:
										_SCREAMING_SNAKE_CASE   :		Tuple       		=    generate_config.eos_token_id
										_SCREAMING_SNAKE_CASE   :		str       		=    generate_config.no_timestamps_token_id
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    generate_config.no_timestamps_token_id + 1

										_SCREAMING_SNAKE_CASE   :		str       		=    decoder_input_length + 1

										if generate_config.is_multilingual:
														# room for language token and task token
														self.begin_index += 2
										if hasattr(lowerCAmelCase_   ,				'max_initial_timestamp_index'							):
														_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    generate_config.max_initial_timestamp_index
										else:
														_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    model_config.vocab_size
										if self.max_initial_timestamp_index is None:
														_SCREAMING_SNAKE_CASE   :		Dict       		=    model_config.vocab_size







						def __call__(    self   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)		->							Dict:
										# suppress <|notimestamps|> which is handled by without_timestamps
										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    scores.at[:, self.no_timestamps_token_id].set(-float('inf'							)							)

										def handle_pairs(lowerCAmelCase_   ,				lowerCAmelCase_							):
														_SCREAMING_SNAKE_CASE   :		int       		=    jnp.where((cur_len - self.begin_index) >= 1   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)
														_SCREAMING_SNAKE_CASE   :		Tuple       		=    jnp.where(
														    input_ids_k[cur_len - 1] >= self.timestamp_begin   ,				True and last_was_timestamp   ,				lowerCAmelCase_   ,				)

														_SCREAMING_SNAKE_CASE   :		Any       		=    jnp.where((cur_len - self.begin_index) < 2   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)
														_SCREAMING_SNAKE_CASE   :		Any       		=    jnp.where(
														    input_ids_k[cur_len - 2] >= self.timestamp_begin   ,				lowerCAmelCase_   ,				lowerCAmelCase_   ,				)

														return jnp.where(
														    lowerCAmelCase_   ,				jnp.where(
														        penultimate_was_timestamp > 0   ,				scores_k.at[self.timestamp_begin :].set(-float('inf'							)							)   ,				scores_k.at[: self.eos_token_id].set(-float('inf'							)							)   ,				)   ,				lowerCAmelCase_   ,				)

										_SCREAMING_SNAKE_CASE   :		Optional[Any]       		=    jax.vmap(lowerCAmelCase_							)(lowerCAmelCase_   ,				lowerCAmelCase_							)

										_SCREAMING_SNAKE_CASE   :		List[str]       		=    jnp.where(cur_len == self.begin_index   ,				lowerCAmelCase_   ,				lowerCAmelCase_							)
										_SCREAMING_SNAKE_CASE   :		Optional[int]       		=    jnp.where(
										    self.max_initial_timestamp_index is not None   ,				True and apply_max_initial_timestamp   ,				lowerCAmelCase_   ,				)

										_SCREAMING_SNAKE_CASE   :		Tuple       		=    self.timestamp_begin + self.max_initial_timestamp_index

										_SCREAMING_SNAKE_CASE   :		Union[str, Any]       		=    jnp.where(
										    lowerCAmelCase_   ,				scores.at[:, last_allowed + 1 :].set(-float('inf'							)							)   ,				lowerCAmelCase_   ,				)

										# if sum of probability over timestamps is above any other token, sample timestamp
										_SCREAMING_SNAKE_CASE   :		str       		=    jax.nn.log_softmax(lowerCAmelCase_   ,				axis=-1							)

										def handle_cumulative_probs(lowerCAmelCase_   ,				lowerCAmelCase_							):
														_SCREAMING_SNAKE_CASE   :		Any       		=    jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]   ,				axis=-1							)
														_SCREAMING_SNAKE_CASE   :		int       		=    jnp.max(logprobs_k[: self.timestamp_begin]							)
														return jnp.where(
														    timestamp_logprob > max_text_token_logprob   ,				scores_k.at[: self.timestamp_begin].set(-float('inf'							)							)   ,				lowerCAmelCase_   ,				)

										_SCREAMING_SNAKE_CASE   :		Dict       		=    jax.vmap(lowerCAmelCase_							)(lowerCAmelCase_   ,				lowerCAmelCase_							)

										return scores

