


"""simple docstring"""


import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase=False ):

			'''simple docstring'''




			__lowerCAmelCase							       =     []
			for i in range(config.num_hidden_layers ):
						# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
						rename_keys.append((f"module.blocks.{i}.norm1.weight", f"vit.encoder.layer.{i}.layernorm_before.weight") )
						rename_keys.append((f"module.blocks.{i}.norm1.bias", f"vit.encoder.layer.{i}.layernorm_before.bias") )
						rename_keys.append(
						    (f"module.blocks.{i}.attn.proj.weight", f"vit.encoder.layer.{i}.attention.output.dense.weight") )
						rename_keys.append((f"module.blocks.{i}.attn.proj.bias", f"vit.encoder.layer.{i}.attention.output.dense.bias") )
						rename_keys.append((f"module.blocks.{i}.norm2.weight", f"vit.encoder.layer.{i}.layernorm_after.weight") )
						rename_keys.append((f"module.blocks.{i}.norm2.bias", f"vit.encoder.layer.{i}.layernorm_after.bias") )
						rename_keys.append((f"module.blocks.{i}.mlp.fc1.weight", f"vit.encoder.layer.{i}.intermediate.dense.weight") )
						rename_keys.append((f"module.blocks.{i}.mlp.fc1.bias", f"vit.encoder.layer.{i}.intermediate.dense.bias") )
						rename_keys.append((f"module.blocks.{i}.mlp.fc2.weight", f"vit.encoder.layer.{i}.output.dense.weight") )
						rename_keys.append((f"module.blocks.{i}.mlp.fc2.bias", f"vit.encoder.layer.{i}.output.dense.bias") )

			# projection layer + position embeddings
			rename_keys.extend(
			    [
			        ("module.cls_token", "vit.embeddings.cls_token"),
			        ("module.patch_embed.proj.weight", "vit.embeddings.patch_embeddings.projection.weight"),
			        ("module.patch_embed.proj.bias", "vit.embeddings.patch_embeddings.projection.bias"),
			        ("module.pos_embed", "vit.embeddings.position_embeddings"),
			    ] )

			if base_model:
						# layernorm + pooler
						rename_keys.extend(
						    [
						        ("module.norm.weight", "layernorm.weight"),
						        ("module.norm.bias", "layernorm.bias"),
						    ] )

						# if just the base model, we should remove "vit" from all keys that start with "vit"
						__lowerCAmelCase							       =     [(pair[0], pair[1][4:]) if pair[1].startswith("vit" ) else pair for pair in rename_keys]
			else:
						# layernorm + classification head
						rename_keys.extend(
						    [
						        ("norm.weight", "vit.layernorm.weight"),
						        ("norm.bias", "vit.layernorm.bias"),
						        ("head.weight", "classifier.weight"),
						        ("head.bias", "classifier.bias"),
						    ] )

			return rename_keys


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase       ,					_UpperCamelCase=False ):

			'''simple docstring'''




			for i in range(config.num_hidden_layers ):
						if base_model:
									__lowerCAmelCase							       =     ""
						else:
									__lowerCAmelCase							       =     "vit."
						# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
						__lowerCAmelCase							       =     state_dict.pop(f"module.blocks.{i}.attn.qkv.weight" )
						__lowerCAmelCase							       =     state_dict.pop(f"module.blocks.{i}.attn.qkv.bias" )
						# next, add query, keys and values (in that order) to the state dict
						__lowerCAmelCase							       =     in_proj_weight[
						    : config.hidden_size, :
						]
						__lowerCAmelCase							       =     in_proj_bias[: config.hidden_size]
						__lowerCAmelCase							       =     in_proj_weight[
						    config.hidden_size : config.hidden_size * 2, :
						]
						__lowerCAmelCase							       =     in_proj_bias[
						    config.hidden_size : config.hidden_size * 2
						]
						__lowerCAmelCase							       =     in_proj_weight[
						    -config.hidden_size :, :
						]
						__lowerCAmelCase							       =     in_proj_bias[-config.hidden_size :]


def _lowerCamelCase      (						_UpperCamelCase ):

			'''simple docstring'''




			__lowerCAmelCase							       =     ["head.weight", "head.bias"]
			for k in ignore_keys:
						state_dict.pop(_UpperCamelCase       ,					_UpperCamelCase )


def _lowerCamelCase      (						_UpperCamelCase ):

			'''simple docstring'''




			__lowerCAmelCase							       =     [
			    "module.fc.fc1.weight",
			    "module.fc.fc1.bias",
			    "module.fc.bn1.weight",
			    "module.fc.bn1.bias",
			    "module.fc.bn1.running_mean",
			    "module.fc.bn1.running_var",
			    "module.fc.bn1.num_batches_tracked",
			    "module.fc.fc2.weight",
			    "module.fc.fc2.bias",
			    "module.fc.bn2.weight",
			    "module.fc.bn2.bias",
			    "module.fc.bn2.running_mean",
			    "module.fc.bn2.running_var",
			    "module.fc.bn2.num_batches_tracked",
			    "module.fc.fc3.weight",
			    "module.fc.fc3.bias",
			]
			for k in ignore_keys:
						state_dict.pop(_UpperCamelCase       ,					_UpperCamelCase )


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			__lowerCAmelCase							       =     dct.pop(_UpperCamelCase )
			__lowerCAmelCase							       =     val


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			__lowerCAmelCase							       =     ViTMSNConfig()
			__lowerCAmelCase							       =     1000

			__lowerCAmelCase							       =     "datasets/huggingface/label-files"
			__lowerCAmelCase							       =     "imagenet-1k-id2label.json"
			__lowerCAmelCase							       =     json.load(open(hf_hub_download(_UpperCamelCase       ,					_UpperCamelCase )       ,					"r" ) )
			__lowerCAmelCase							       =     {int(_UpperCamelCase ): v for k, v in idalabel.items()}
			__lowerCAmelCase							       =     idalabel
			__lowerCAmelCase							       =     {v: k for k, v in idalabel.items()}

			if "s16" in checkpoint_url:
						__lowerCAmelCase							       =     384
						__lowerCAmelCase							       =     1536
						__lowerCAmelCase							       =     6
			elif "l16" in checkpoint_url:
						__lowerCAmelCase							       =     1024
						__lowerCAmelCase							       =     4096
						__lowerCAmelCase							       =     24
						__lowerCAmelCase							       =     16
						__lowerCAmelCase							       =     0.1
			elif "b4" in checkpoint_url:
						__lowerCAmelCase							       =     4
			elif "l7" in checkpoint_url:
						__lowerCAmelCase							       =     7
						__lowerCAmelCase							       =     1024
						__lowerCAmelCase							       =     4096
						__lowerCAmelCase							       =     24
						__lowerCAmelCase							       =     16
						__lowerCAmelCase							       =     0.1

			__lowerCAmelCase							       =     ViTMSNModel(_UpperCamelCase )

			__lowerCAmelCase							       =     torch.hub.load_state_dict_from_url(_UpperCamelCase       ,					map_location="cpu" )["target_encoder"]

			__lowerCAmelCase							       =     ViTImageProcessor(size=config.image_size )

			remove_projection_head(_UpperCamelCase )
			__lowerCAmelCase							       =     create_rename_keys(_UpperCamelCase       ,					base_model=_UpperCamelCase )

			for src, dest in rename_keys:
						rename_key(_UpperCamelCase       ,					_UpperCamelCase       ,					_UpperCamelCase )
			read_in_q_k_v(_UpperCamelCase       ,					_UpperCamelCase       ,					base_model=_UpperCamelCase )

			model.load_state_dict(_UpperCamelCase )
			model.eval()

			__lowerCAmelCase							       =     "http://images.cocodataset.org/val2017/000000039769.jpg"

			__lowerCAmelCase							       =     Image.open(requests.get(_UpperCamelCase       ,					stream=_UpperCamelCase ).raw )
			__lowerCAmelCase							       =     ViTImageProcessor(
			    size=config.image_size       ,					image_mean=_UpperCamelCase       ,					image_std=_UpperCamelCase )
			__lowerCAmelCase							       =     image_processor(images=_UpperCamelCase       ,					return_tensors="pt" )

			# forward pass
			torch.manual_seed(2 )
			__lowerCAmelCase							       =     model(**_UpperCamelCase )
			__lowerCAmelCase							       =     outputs.last_hidden_state

			# The following Colab Notebook was used to generate these outputs:
			# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
			if "s16" in checkpoint_url:
						__lowerCAmelCase							       =     torch.tensor([[-1.09_15, -1.48_76, -1.18_09]] )
			elif "b16" in checkpoint_url:
						__lowerCAmelCase							       =     torch.tensor([[14.28_89, -18.90_45, 11.72_81]] )
			elif "l16" in checkpoint_url:
						__lowerCAmelCase							       =     torch.tensor([[41.50_28, -22.86_81, 45.64_75]] )
			elif "b4" in checkpoint_url:
						__lowerCAmelCase							       =     torch.tensor([[-4.38_68, 5.29_32, -0.41_37]] )
			else:
						__lowerCAmelCase							       =     torch.tensor([[-0.17_92, -0.64_65, 2.42_63]] )

			# verify logits
			assert torch.allclose(last_hidden_state[:, 0, :3]       ,					_UpperCamelCase       ,					atol=1e-4 )

			print(f"Saving model to {pytorch_dump_folder_path}" )
			model.save_pretrained(_UpperCamelCase )

			print(f"Saving image processor to {pytorch_dump_folder_path}" )
			image_processor.save_pretrained(_UpperCamelCase )


if __name__ == "__main__":
	A	:  List[str]            =     argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    "--checkpoint_url",
	    default="https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar",
	    type=str,
	    help="URL of the checkpoint you'd like to convert.",
	)
	parser.add_argument(
	    "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
	)

	A	:  Union[str, Any]            =     parser.parse_args()
	convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)








"""simple docstring"""


import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


A	:  Dict            =     get_logger(__name__)


A	:  Dict            =     R"\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n"
class     _UpperCamelCase    :
							'''simple docstring'''







							@add_start_docstrings(__a      )
							def __call__(	self       ,  __a       ,  __a      ):
										raise NotImplementedError(
										    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."      )
class     _UpperCamelCase    :
							'''simple docstring'''







							@add_start_docstrings(__a      )
							def __call__(	self       ,  __a       ,  __a      ):
										raise NotImplementedError(
										    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."      )
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							@add_start_docstrings(__a      )
							def __call__(	self       ,  __a       ,  __a       ,  __a       ,  **__a      ):
										for processor in self:
													__lowerCAmelCase							       =     inspect.signature(processor.__call__      ).parameters
													if len(__a      ) > 3:
																if not all(arg in kwargs for arg in list(function_args.keys()      )[2:]      ):
																			raise ValueError(
																			    f"Make sure that all the required parameters: {list(function_args.keys()      )} for "
																			    f"{processor.__class__} are passed to the logits processor."      )
																__lowerCAmelCase							       =     processor(__a       ,  __a       ,  __a       ,  **__a      )
													else:
																__lowerCAmelCase							       =     processor(__a       ,  __a       ,  __a      )
										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a      ):
										if not isinstance(__a       ,  __a      ) or not (temperature > 0):
													raise ValueError(f"`temperature` has to be a strictly positive float, but is {temperature}"      )

										__lowerCAmelCase							       =     temperature



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     scores / self.temperature
										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a = -float("Inf"      )       ,  __a = 1      ):
										if not isinstance(__a       ,  __a      ) or (top_p < 0 or top_p > 1.0):
													raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}"      )
										if not isinstance(__a       ,  __a      ) or (min_tokens_to_keep < 1):
													raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"      )

										__lowerCAmelCase							       =     top_p
										__lowerCAmelCase							       =     filter_value
										__lowerCAmelCase							       =     min_tokens_to_keep



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     lax.top_k(__a       ,  scores.shape[-1]      )

										__lowerCAmelCase							       =     jnp.full_like(__a       ,  self.filter_value      )
										__lowerCAmelCase							       =     jax.nn.softmax(__a       ,  axis=-1      ).cumsum(axis=-1      )
										__lowerCAmelCase							       =     cumulative_probs < self.top_p

										# include the token that is higher than top_p as well
										__lowerCAmelCase							       =     jnp.roll(__a       ,  1      )
										score_mask |= score_mask.at[:, 0].set(__a      )

										# min tokens to keep
										__lowerCAmelCase							       =     score_mask.at[:, : self.min_tokens_to_keep].set(__a      )

										__lowerCAmelCase							       =     jnp.where(__a       ,  __a       ,  __a      )
										__lowerCAmelCase							       =     jax.lax.sort_key_val(__a       ,  __a      )[-1]

										return next_scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a = -float("Inf"      )       ,  __a = 1      ):
										if not isinstance(__a       ,  __a      ) or top_k <= 0:
													raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}"      )

										__lowerCAmelCase							       =     max(__a       ,  __a      )
										__lowerCAmelCase							       =     filter_value



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     scores.shape
										__lowerCAmelCase							       =     jnp.full(batch_size * vocab_size       ,  self.filter_value      )

										__lowerCAmelCase							       =     min(self.top_k       ,  scores.shape[-1]      )  # Safety check
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     lax.top_k(__a       ,  __a      )
										__lowerCAmelCase							       =     jnp.broadcast_to((jnp.arange(__a      ) * vocab_size)[:, None]       ,  (batch_size, topk)      ).flatten()
										__lowerCAmelCase							       =     topk_scores.flatten()
										__lowerCAmelCase							       =     topk_indices.flatten() + shift

										__lowerCAmelCase							       =     next_scores_flat.at[topk_indices_flat].set(__a      )
										__lowerCAmelCase							       =     next_scores_flat.reshape(__a       ,  __a      )
										return next_scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a      ):
										__lowerCAmelCase							       =     bos_token_id



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     jnp.full(scores.shape       ,  -float("inf"      )      )

										__lowerCAmelCase							       =     1 - jnp.bool_(cur_len - 1      )

										__lowerCAmelCase							       =     jnp.where(__a       ,  new_scores.at[:, self.bos_token_id].set(0      )       ,  __a      )

										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     max_length
										__lowerCAmelCase							       =     eos_token_id



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     jnp.full(scores.shape       ,  -float("inf"      )      )

										__lowerCAmelCase							       =     1 - jnp.bool_(cur_len - self.max_length + 1      )

										__lowerCAmelCase							       =     jnp.where(__a       ,  new_scores.at[:, self.eos_token_id].set(0      )       ,  __a      )

										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a      ):
										if not isinstance(__a       ,  __a      ) or min_length < 0:
													raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}"      )

										if not isinstance(__a       ,  __a      ) or eos_token_id < 0:
													raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}"      )

										__lowerCAmelCase							       =     min_length
										__lowerCAmelCase							       =     eos_token_id



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										# create boolean flag to decide if min length penalty should be applied
										__lowerCAmelCase							       =     1 - jnp.clip(cur_len - self.min_length       ,  0       ,  1      )

										__lowerCAmelCase							       =     jnp.where(__a       ,  scores.at[:, self.eos_token_id].set(-float("inf"      )      )       ,  __a      )

										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     list(__a      )
										__lowerCAmelCase							       =     begin_index



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     1 - jnp.bool_(cur_len - self.begin_index      )

										__lowerCAmelCase							       =     jnp.where(__a       ,  scores.at[:, self.begin_suppress_tokens].set(-float("inf"      )      )       ,  __a      )

										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a      ):
										__lowerCAmelCase							       =     list(__a      )



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     scores.at[..., self.suppress_tokens].set(-float("inf"      )      )

										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a      ):
										__lowerCAmelCase							       =     dict(__a      )
										# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
										# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
										# Indexes without forced tokens will have a negative value.
										__lowerCAmelCase							       =     jnp.ones((max(force_token_map.keys()      ) + 1)       ,  dtype=jnp.intaa      ) * -1
										for index, token in force_token_map.items():
													if token is not None:
																__lowerCAmelCase							       =     force_token_array.at[index].set(__a      )
										__lowerCAmelCase							       =     jnp.intaa(__a      )



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										def _force_token(__a      ):
													__lowerCAmelCase							       =     scores.shape[0]
													__lowerCAmelCase							       =     self.force_token_array[generation_idx]

													__lowerCAmelCase							       =     jnp.ones_like(__a       ,  dtype=scores.dtype      ) * -float("inf"      )
													__lowerCAmelCase							       =     jnp.zeros((batch_size, 1)       ,  dtype=scores.dtype      )
													__lowerCAmelCase							       =     lax.dynamic_update_slice(__a       ,  __a       ,  (0, current_token)      )
													return new_scores

										__lowerCAmelCase							       =     lax.cond(
										    cur_len >= self.force_token_array.shape[0]       ,  lambda: scores       ,  lambda: lax.cond(
										        self.force_token_array[cur_len] >= 0       ,  lambda: _force_token(__a      )       ,  lambda: scores       ,  )       ,  )
										return scores
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     generate_config.eos_token_id
										__lowerCAmelCase							       =     generate_config.no_timestamps_token_id
										__lowerCAmelCase							       =     generate_config.no_timestamps_token_id + 1

										__lowerCAmelCase							       =     decoder_input_length + 1

										if generate_config.is_multilingual:
													# room for language token and task token
													self.begin_index += 2
										if hasattr(__a       ,  "max_initial_timestamp_index"      ):
													__lowerCAmelCase							       =     generate_config.max_initial_timestamp_index
										else:
													__lowerCAmelCase							       =     model_config.vocab_size
										if self.max_initial_timestamp_index is None:
													__lowerCAmelCase							       =     model_config.vocab_size



							def __call__(	self       ,  __a       ,  __a       ,  __a      ):
										# suppress <|notimestamps|> which is handled by without_timestamps
										__lowerCAmelCase							       =     scores.at[:, self.no_timestamps_token_id].set(-float("inf"      )      )

										def handle_pairs(__a       ,  __a      ):
													__lowerCAmelCase							       =     jnp.where((cur_len - self.begin_index) >= 1       ,  __a       ,  __a      )
													__lowerCAmelCase							       =     jnp.where(
													    input_ids_k[cur_len - 1] >= self.timestamp_begin       ,  True and last_was_timestamp       ,  __a       ,  )

													__lowerCAmelCase							       =     jnp.where((cur_len - self.begin_index) < 2       ,  __a       ,  __a      )
													__lowerCAmelCase							       =     jnp.where(
													    input_ids_k[cur_len - 2] >= self.timestamp_begin       ,  __a       ,  __a       ,  )

													return jnp.where(
													    __a       ,  jnp.where(
													        penultimate_was_timestamp > 0       ,  scores_k.at[self.timestamp_begin :].set(-float("inf"      )      )       ,  scores_k.at[: self.eos_token_id].set(-float("inf"      )      )       ,  )       ,  __a       ,  )

										__lowerCAmelCase							       =     jax.vmap(__a      )(__a       ,  __a      )

										__lowerCAmelCase							       =     jnp.where(cur_len == self.begin_index       ,  __a       ,  __a      )
										__lowerCAmelCase							       =     jnp.where(
										    self.max_initial_timestamp_index is not None       ,  True and apply_max_initial_timestamp       ,  __a       ,  )

										__lowerCAmelCase							       =     self.timestamp_begin + self.max_initial_timestamp_index

										__lowerCAmelCase							       =     jnp.where(
										    __a       ,  scores.at[:, last_allowed + 1 :].set(-float("inf"      )      )       ,  __a       ,  )

										# if sum of probability over timestamps is above any other token, sample timestamp
										__lowerCAmelCase							       =     jax.nn.log_softmax(__a       ,  axis=-1      )

										def handle_cumulative_probs(__a       ,  __a      ):
													__lowerCAmelCase							       =     jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]       ,  axis=-1      )
													__lowerCAmelCase							       =     jnp.max(logprobs_k[: self.timestamp_begin]      )
													return jnp.where(
													    timestamp_logprob > max_text_token_logprob       ,  scores_k.at[: self.timestamp_begin].set(-float("inf"      )      )       ,  __a       ,  )

										__lowerCAmelCase							       =     jax.vmap(__a      )(__a       ,  __a      )

										return scores




