


import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin





class       A    (   A_					,				A_ ):


							@register_to_config
							def __init__(self      ,				lowerCAmelCase = 1_2_8      ,				lowerCAmelCase = 2_5_6      ,				lowerCAmelCase = 20_00.0      ,				lowerCAmelCase = 7_6_8      ,				lowerCAmelCase = 1_2      ,				lowerCAmelCase = 1_2      ,				lowerCAmelCase = 6_4      ,				lowerCAmelCase = 2_0_4_8      ,				lowerCAmelCase = 0.1      ,				):
									super().__init__()

									__lowercase=			nn.Sequential(
									    nn.Linear(lowerCAmelCase      ,				d_model * 4      ,				bias=lowerCAmelCase					)      ,				nn.SiLU()      ,				nn.Linear(d_model * 4      ,				d_model * 4      ,				bias=lowerCAmelCase					)      ,				nn.SiLU()      ,				)

									__lowercase=			nn.Embedding(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			False

									__lowercase=			nn.Linear(lowerCAmelCase      ,				lowerCAmelCase      ,				bias=lowerCAmelCase					)

									__lowercase=			nn.Dropout(p=lowerCAmelCase					)

									__lowercase=			nn.ModuleList()
									for lyr_num in range(lowerCAmelCase					):
											# FiLM conditional T5 decoder
											__lowercase=			DecoderLayer(d_model=lowerCAmelCase      ,				d_kv=lowerCAmelCase      ,				num_heads=lowerCAmelCase      ,				d_ff=lowerCAmelCase      ,				dropout_rate=lowerCAmelCase					)
											self.decoders.append(lowerCAmelCase					)

									__lowercase=			TaLayerNorm(lowerCAmelCase					)

									__lowercase=			nn.Dropout(p=lowerCAmelCase					)
									__lowercase=			nn.Linear(lowerCAmelCase      ,				lowerCAmelCase      ,				bias=lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase=			torch.mul(query_input.unsqueeze(-1					)      ,				key_input.unsqueeze(-2					)					)
									return mask.unsqueeze(-3					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase,							__lowercase,							__lowercase=			decoder_input_tokens.shape
									assert decoder_noise_time.shape == (batch,)

									# decoder_noise_time is in [0, 1), so rescale to expected timing range.
									__lowercase=			get_timestep_embedding(
									    decoder_noise_time * self.config.max_decoder_noise_time      ,				embedding_dim=self.config.d_model      ,				max_period=self.config.max_decoder_noise_time      ,				).to(dtype=self.dtype					)

									__lowercase=			self.conditioning_emb(lowerCAmelCase					).unsqueeze(1					)

									assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

									__lowercase=			decoder_input_tokens.shape[1]

									# If we want to use relative positions for audio context, we can just offset
									# this sequence by the length of encodings_and_masks.
									__lowercase=			torch.broadcast_to(
									    torch.arange(lowerCAmelCase      ,				device=decoder_input_tokens.device					)      ,				(batch, seq_length)      ,				)

									__lowercase=			self.position_encoding(lowerCAmelCase					)

									__lowercase=			self.continuous_inputs_projection(lowerCAmelCase					)
									inputs += position_encodings
									__lowercase=			self.dropout(lowerCAmelCase					)

									# decoder: No padding present.
									__lowercase=			torch.ones(
									    decoder_input_tokens.shape[:2]      ,				device=decoder_input_tokens.device      ,				dtype=inputs.dtype					)

									# Translate encoding masks to encoder-decoder masks.
									__lowercase=			[(x, self.encoder_decoder_mask(lowerCAmelCase      ,				lowerCAmelCase					)) for x, y in encodings_and_masks]

									# cross attend style: concat encodings
									__lowercase=			torch.cat([x[0] for x in encodings_and_encdec_masks]      ,				dim=1					)
									__lowercase=			torch.cat([x[1] for x in encodings_and_encdec_masks]      ,				dim=-1					)

									for lyr in self.decoders:
											__lowercase=			lyr(
											    lowerCAmelCase      ,				conditioning_emb=lowerCAmelCase      ,				encoder_hidden_states=lowerCAmelCase      ,				encoder_attention_mask=lowerCAmelCase      ,				)[0]

									__lowercase=			self.decoder_norm(lowerCAmelCase					)
									__lowercase=			self.post_dropout(lowerCAmelCase					)

									__lowercase=			self.spec_out(lowerCAmelCase					)
									return spec_out





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase=1E-6					):
									super().__init__()
									__lowercase=			nn.ModuleList()

									# cond self attention: layer 0
									self.layer.append(
									    TaLayerSelfAttentionCond(d_model=lowerCAmelCase      ,				d_kv=lowerCAmelCase      ,				num_heads=lowerCAmelCase      ,				dropout_rate=lowerCAmelCase					)					)

									# cross attention: layer 1
									self.layer.append(
									    TaLayerCrossAttention(
									        d_model=lowerCAmelCase      ,				d_kv=lowerCAmelCase      ,				num_heads=lowerCAmelCase      ,				dropout_rate=lowerCAmelCase      ,				layer_norm_epsilon=lowerCAmelCase      ,				)					)

									# Film Cond MLP + dropout: last layer
									self.layer.append(
									    TaLayerFFCond(d_model=lowerCAmelCase      ,				d_ff=lowerCAmelCase      ,				dropout_rate=lowerCAmelCase      ,				layer_norm_epsilon=lowerCAmelCase					)					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				):
									__lowercase=			self.layer[0](
									    lowerCAmelCase      ,				conditioning_emb=lowerCAmelCase      ,				attention_mask=lowerCAmelCase      ,				)

									if encoder_hidden_states is not None:
											__lowercase=			torch.where(encoder_attention_mask > 0      ,				0      ,				-1E10					).to(
											    encoder_hidden_states.dtype					)

											__lowercase=			self.layer[1](
											    lowerCAmelCase      ,				key_value_states=lowerCAmelCase      ,				attention_mask=lowerCAmelCase      ,				)

									# Apply Film Conditional Feed Forward layer
									__lowercase=			self.layer[-1](lowerCAmelCase      ,				lowerCAmelCase					)

									return (hidden_states,)





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									super().__init__()
									__lowercase=			TaLayerNorm(lowerCAmelCase					)
									__lowercase=			TaFiLMLayer(in_features=d_model * 4      ,				out_features=lowerCAmelCase					)
									__lowercase=			Attention(query_dim=lowerCAmelCase      ,				heads=lowerCAmelCase      ,				dim_head=lowerCAmelCase      ,				out_bias=lowerCAmelCase      ,				scale_qk=lowerCAmelCase					)
									__lowercase=			nn.Dropout(lowerCAmelCase					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				):
									# pre_self_attention_layer_norm
									__lowercase=			self.layer_norm(lowerCAmelCase					)

									if conditioning_emb is not None:
											__lowercase=			self.FiLMLayer(lowerCAmelCase      ,				lowerCAmelCase					)

									# Self-attention block
									__lowercase=			self.attention(lowerCAmelCase					)

									__lowercase=			hidden_states + self.dropout(lowerCAmelCase					)

									return hidden_states





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									super().__init__()
									__lowercase=			Attention(query_dim=lowerCAmelCase      ,				heads=lowerCAmelCase      ,				dim_head=lowerCAmelCase      ,				out_bias=lowerCAmelCase      ,				scale_qk=lowerCAmelCase					)
									__lowercase=			TaLayerNorm(lowerCAmelCase      ,				eps=lowerCAmelCase					)
									__lowercase=			nn.Dropout(lowerCAmelCase					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None      ,				lowerCAmelCase=None      ,				):
									__lowercase=			self.layer_norm(lowerCAmelCase					)
									__lowercase=			self.attention(
									    lowerCAmelCase      ,				encoder_hidden_states=lowerCAmelCase      ,				attention_mask=attention_mask.squeeze(1					)      ,				)
									__lowercase=			hidden_states + self.dropout(lowerCAmelCase					)
									return layer_output





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									super().__init__()
									__lowercase=			TaDenseGatedActDense(d_model=lowerCAmelCase      ,				d_ff=lowerCAmelCase      ,				dropout_rate=lowerCAmelCase					)
									__lowercase=			TaFiLMLayer(in_features=d_model * 4      ,				out_features=lowerCAmelCase					)
									__lowercase=			TaLayerNorm(lowerCAmelCase      ,				eps=lowerCAmelCase					)
									__lowercase=			nn.Dropout(lowerCAmelCase					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None					):
									__lowercase=			self.layer_norm(lowerCAmelCase					)
									if conditioning_emb is not None:
											__lowercase=			self.film(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			self.DenseReluDense(lowerCAmelCase					)
									__lowercase=			hidden_states + self.dropout(lowerCAmelCase					)
									return hidden_states





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									super().__init__()
									__lowercase=			nn.Linear(lowerCAmelCase      ,				lowerCAmelCase      ,				bias=lowerCAmelCase					)
									__lowercase=			nn.Linear(lowerCAmelCase      ,				lowerCAmelCase      ,				bias=lowerCAmelCase					)
									__lowercase=			nn.Linear(lowerCAmelCase      ,				lowerCAmelCase      ,				bias=lowerCAmelCase					)
									__lowercase=			nn.Dropout(lowerCAmelCase					)
									__lowercase=			NewGELUActivation()







							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			self.act(self.wi_a(lowerCAmelCase					)					)
									__lowercase=			self.wi_a(lowerCAmelCase					)
									__lowercase=			hidden_gelu * hidden_linear
									__lowercase=			self.dropout(lowerCAmelCase					)

									__lowercase=			self.wo(lowerCAmelCase					)
									return hidden_states





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase=1E-6					):
									super().__init__()
									__lowercase=			nn.Parameter(torch.ones(lowerCAmelCase					)					)
									__lowercase=			eps







							def 			_A       (self      ,				lowerCAmelCase					):
									# T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
									# Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
									# w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
									# half-precision inputs is done in fp32

									__lowercase=			hidden_states.to(torch.floataa					).pow(2					).mean(-1      ,				keepdim=lowerCAmelCase					)
									__lowercase=			hidden_states * torch.rsqrt(variance + self.variance_epsilon					)

									# convert into half-precision if necessary
									if self.weight.dtype in [torch.floataa, torch.bfloataa]:
											__lowercase=			hidden_states.to(self.weight.dtype					)

									return self.weight * hidden_states





class       A    (   nn.Module ):


							def 			_A       (self      ,				lowerCAmelCase					):
									return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi					) * (input + 0.04_47_15 * torch.pow(lowerCAmelCase      ,				3.0					))					))





class       A    (   nn.Module ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									super().__init__()
									__lowercase=			nn.Linear(lowerCAmelCase      ,				out_features * 2      ,				bias=lowerCAmelCase					)







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase=			self.scale_bias(lowerCAmelCase					)
									__lowercase,							__lowercase=			torch.chunk(lowerCAmelCase      ,				2      ,				-1					)
									__lowercase=			x * (1 + scale) + shift
									return x










import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase           =		'''▁'''

lowerCAmelCase           =		{'''vocab_file''': '''spiece.model'''}

lowerCAmelCase           =		{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

lowerCAmelCase           =		{
    '''google/pegasus-xsum''': 5_1_2,
}


lowerCAmelCase           =		logging.get_logger(__name__)





class       A    (   A_ ):
							UpperCamelCase_   :							Union[str, Any]       =VOCAB_FILES_NAMES

							UpperCamelCase_   :							List[Any]       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							int       =PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							Tuple       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							int       =['''input_ids''', '''attention_mask''']


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase="<pad>"      ,				lowerCAmelCase="</s>"      ,				lowerCAmelCase="<unk>"      ,				lowerCAmelCase="<mask_2>"      ,				lowerCAmelCase="<mask_1>"      ,				lowerCAmelCase=None      ,				lowerCAmelCase=1_0_3      ,				lowerCAmelCase = None      ,				**lowerCAmelCase      ,				):
									__lowercase=			offset
									if additional_special_tokens is not None:
											if not isinstance(lowerCAmelCase      ,				lowerCAmelCase					):
													raise TypeError(
													    f'additional_special_tokens should be of type {type(lowerCAmelCase					)}, but is'
													    f' {type(lowerCAmelCase					)}'					)

											__lowercase=			(
											    ([mask_token_sent] + additional_special_tokens)
											    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
											    else additional_special_tokens
											)
											# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
											additional_special_tokens_extended += [
											    f'<unk_{i}>' for i in range(len(lowerCAmelCase					)      ,				self.offset - 1					)
											]

											if len(set(lowerCAmelCase					)					) != len(lowerCAmelCase					):
													raise ValueError(
													    'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
													    f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'					)
											__lowercase=			additional_special_tokens_extended
									else:
											__lowercase=			[mask_token_sent] if mask_token_sent is not None else []
											additional_special_tokens += [f'<unk_{i}>' for i in range(2      ,				self.offset					)]

									__lowercase=			{} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    eos_token=lowerCAmelCase      ,				unk_token=lowerCAmelCase      ,				mask_token=lowerCAmelCase      ,				pad_token=lowerCAmelCase      ,				mask_token_sent=lowerCAmelCase      ,				offset=lowerCAmelCase      ,				additional_special_tokens=lowerCAmelCase      ,				sp_model_kwargs=self.sp_model_kwargs      ,				**lowerCAmelCase      ,				)
									__lowercase=			mask_token_sent
									__lowercase=			vocab_file
									__lowercase=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(lowerCAmelCase					)

									# add special tokens to encoder dict
									__lowercase=			{
									    0: self.pad_token,
									    1: self.eos_token,
									}

									if self.mask_token_sent is not None:
											self.encoder.update(
											    {
											        2: self.mask_token_sent,
											        3: self.mask_token,
											    }					)

									if self.offset > 0:
											# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
											# mask_token_sent is already added to list -> so start at 1
											self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,				self.offset - 1					)}					)

									__lowercase=			{v: k for k, v in self.encoder.items()}


							@property
							def 			_A       (self					):
									return len(self.sp_model					) + self.offset


							def 			_A       (self					):
									__lowercase=			{self.convert_ids_to_tokens(lowerCAmelCase					): i for i in range(self.vocab_size					)}
									vocab.update(self.added_tokens_encoder					)
									return vocab


							def __getstate__(self					):
									__lowercase=			self.__dict__.copy()
									__lowercase=			None
									return state


							def __setstate__(self      ,				lowerCAmelCase					):
									__lowercase=			d

									# for backward compatibility
									if not hasattr(self      ,				'sp_model_kwargs'					):
											__lowercase=			{}

									__lowercase=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(self.vocab_file					)


							def 			_A       (self      ,				lowerCAmelCase					):
									return self.sp_model.encode(lowerCAmelCase      ,				out_type=lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase					):
									if token in self.decoder:
											return self.decoder[token]
									elif token in self.added_tokens_decoder:
											return self.added_tokens_decoder[token]
									__lowercase=			self.sp_model.piece_to_id(lowerCAmelCase					)
									return sp_id + self.offset


							def 			_A       (self      ,				lowerCAmelCase					):
									if index in self.encoder:
											return self.encoder[index]
									elif index in self.added_tokens_encoder:
											return self.added_tokens_encoder[index]
									else:
											__lowercase=			self.sp_model.IdToPiece(index - self.offset					)
									return token


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			[]
									__lowercase=			''
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													out_string += self.sp_model.decode(lowerCAmelCase					) + token
													__lowercase=			[]
											else:
													current_sub_tokens.append(lowerCAmelCase					)
									out_string += self.sp_model.decode(lowerCAmelCase					)
									return out_string.strip()


							def 			_A       (self      ,				lowerCAmelCase=False					):
									return 1


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			set(self.all_special_ids					)  # call it once instead of inside list comp
									all_special_ids.remove(self.unk_token_id					)  # <unk> is only sometimes special

									return [1 if x in all_special_ids else 0 for x in seq]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None      ,				lowerCAmelCase = False					):
									if already_has_special_tokens:
											return self._special_token_mask(lowerCAmelCase					)
									elif token_ids_a is None:
											return self._special_token_mask(lowerCAmelCase					) + [1]
									else:
											return self._special_token_mask(token_ids_a + token_ids_a					) + [1]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=None					):
									if token_ids_a is None:
											return token_ids_a + [self.eos_token_id]
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return token_ids_a + token_ids_a + [self.eos_token_id]







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									if not os.path.isdir(lowerCAmelCase					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
									__lowercase=			os.path.join(
									    lowerCAmelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

									if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase					) and os.path.isfile(self.vocab_file					):
											copyfile(self.vocab_file      ,				lowerCAmelCase					)
									elif not os.path.isfile(self.vocab_file					):
											with open(lowerCAmelCase      ,				'wb'					) as fi:
													__lowercase=			self.sp_model.serialized_model_proto()
													fi.write(lowerCAmelCase					)

									return (out_vocab_file,)






