




from __future__ import annotations

import unittest

from transformers import LEDConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import TFLEDForConditionalGeneration, TFLEDModel


@require_tf
class  A		:
		__snake_case									=   LEDConfig
		__snake_case									=   {}
		__snake_case									=   'gelu'



		def __init__(     self,	UpperCamelCase__,	UpperCamelCase__=13,	UpperCamelCase__=7,	UpperCamelCase__=True,	UpperCamelCase__=False,	UpperCamelCase__=99,	UpperCamelCase__=32,	UpperCamelCase__=2,	UpperCamelCase__=4,	UpperCamelCase__=37,	UpperCamelCase__=0.1,	UpperCamelCase__=0.1,	UpperCamelCase__=20,	UpperCamelCase__=2,	UpperCamelCase__=1,	UpperCamelCase__=0,	UpperCamelCase__=4,	):







								"""simple docstring"""



								lowerCAmelCase_      			=  parent
								lowerCAmelCase_      			=  batch_size
								lowerCAmelCase_      			=  seq_length
								lowerCAmelCase_      			=  is_training
								lowerCAmelCase_      			=  use_labels
								lowerCAmelCase_      			=  vocab_size
								lowerCAmelCase_      			=  hidden_size
								lowerCAmelCase_      			=  num_hidden_layers
								lowerCAmelCase_      			=  num_attention_heads
								lowerCAmelCase_      			=  intermediate_size

								lowerCAmelCase_      			=  hidden_dropout_prob
								lowerCAmelCase_      			=  attention_probs_dropout_prob
								lowerCAmelCase_      			=  max_position_embeddings
								lowerCAmelCase_      			=  eos_token_id
								lowerCAmelCase_      			=  pad_token_id
								lowerCAmelCase_      			=  bos_token_id
								lowerCAmelCase_      			=  attention_window

								# `ModelTesterMixin.test_attention_outputs` is expecting attention tensors to be of size
								# [num_attention_heads, encoder_seq_length, encoder_key_length], but TFLongformerSelfAttention
								# returns attention of shape [num_attention_heads, encoder_seq_length, self.attention_window + 1]
								# because its local attention only attends to `self.attention_window` and one before and one after
								lowerCAmelCase_      			=  self.attention_window + 2

								# because of padding `encoder_seq_length`, is different from `seq_length`. Relevant for
								# the `test_attention_outputs` and `test_hidden_states_output` tests
								lowerCAmelCase_      			=  (
								    self.seq_length + (self.attention_window - self.seq_length % self.attention_window) % self.attention_window
								)



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  ids_tensor([self.batch_size, self.seq_length - 1],	self.vocab_size   )
								lowerCAmelCase_      			=  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size   ),	1   )
								lowerCAmelCase_      			=  tf.concat([input_ids, eos_tensor],	axis=1   )

								lowerCAmelCase_      			=  ids_tensor([self.batch_size, self.seq_length],	self.vocab_size   )

								lowerCAmelCase_      			=  self.config_cls(
								    vocab_size=self.vocab_size,	d_model=self.hidden_size,	encoder_layers=self.num_hidden_layers,	decoder_layers=self.num_hidden_layers,	encoder_attention_heads=self.num_attention_heads,	decoder_attention_heads=self.num_attention_heads,	encoder_ffn_dim=self.intermediate_size,	decoder_ffn_dim=self.intermediate_size,	dropout=self.hidden_dropout_prob,	attention_dropout=self.attention_probs_dropout_prob,	max_position_embeddings=self.max_position_embeddings,	eos_token_ids=[2],	bos_token_id=self.bos_token_id,	pad_token_id=self.pad_token_id,	decoder_start_token_id=self.pad_token_id,	attention_window=self.attention_window,	**self.config_updates,	)
								lowerCAmelCase_      			=  prepare_led_inputs_dict(UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   )
								lowerCAmelCase_      			=  tf.concat(
								    [tf.zeros_like(UpperCamelCase__   )[:, :-1], tf.ones_like(UpperCamelCase__   )[:, -1:]],	axis=-1,	)
								lowerCAmelCase_      			=  global_attention_mask
								return config, inputs_dict







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  TFLEDModel(config=UpperCamelCase__   ).get_decoder()
								lowerCAmelCase_      			=  inputs_dict['''input_ids''']

								lowerCAmelCase_      			=  input_ids[:1, :]
								lowerCAmelCase_      			=  inputs_dict['''attention_mask'''][:1, :]
								lowerCAmelCase_      			=  1

								# first forward pass
								lowerCAmelCase_      			=  model(UpperCamelCase__,	attention_mask=UpperCamelCase__,	use_cache=UpperCamelCase__   )

								lowerCAmelCase_			,					lowerCAmelCase_      			=  outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								lowerCAmelCase_      			=  ids_tensor((self.batch_size, 3),	config.vocab_size   )
								lowerCAmelCase_      			=  tf.cast(ids_tensor((self.batch_size, 3),	2   ),	tf.inta   )

								# append to next input_ids and
								lowerCAmelCase_      			=  tf.concat([input_ids, next_tokens],	axis=-1   )
								lowerCAmelCase_      			=  tf.concat([attention_mask, next_attn_mask],	axis=-1   )

								lowerCAmelCase_      			=  model(UpperCamelCase__,	attention_mask=UpperCamelCase__   )[0]
								lowerCAmelCase_      			=  model(UpperCamelCase__,	attention_mask=UpperCamelCase__,	past_key_values=UpperCamelCase__   )[0]

								self.parent.assertEqual(next_tokens.shape[1],	output_from_past.shape[1]   )

								# select random slice
								lowerCAmelCase_      			=  int(ids_tensor((1,),	output_from_past.shape[-1]   )   )
								lowerCAmelCase_      			=  output_from_no_past[:, -3:, random_slice_idx]
								lowerCAmelCase_      			=  output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(UpperCamelCase__,	UpperCamelCase__,	rtol=1E-3   )







def __UpperCamelCase			(       _A ,  _A ,  _A ,  _A=None ,  _A=None ,  _A=None ,  _A=None ,  ):
						if attention_mask is None:
												lowerCAmelCase_      			=  tf.cast(tf.math.not_equal(_A ,  config.pad_token_id  ) ,  tf.inta  )
						if decoder_attention_mask is None:
												lowerCAmelCase_      			=  tf.concat(
												    [
												        tf.ones(decoder_input_ids[:, :1].shape ,  dtype=tf.inta  ),
												        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:] ,  config.pad_token_id  ) ,  tf.inta  ),
												    ] ,  axis=-1 ,  )
						if head_mask is None:
												lowerCAmelCase_      			=  tf.ones((config.encoder_layers, config.encoder_attention_heads)  )
						if decoder_head_mask is None:
												lowerCAmelCase_      			=  tf.ones((config.decoder_layers, config.decoder_attention_heads)  )
						return {
						    "input_ids": input_ids,
						    "attention_mask": attention_mask,
						    "decoder_input_ids": decoder_input_ids,
						    "decoder_attention_mask": decoder_attention_mask,
						    "head_mask": head_mask,
						    "decoder_head_mask": decoder_head_mask,
						}


@require_tf
class  A		(  __UpperCAmelCase      ,				__UpperCAmelCase      ,				unittest.TestCase     ):
		__snake_case									=   (TFLEDForConditionalGeneration, TFLEDModel) if is_tf_available() else ()
		__snake_case									=   (TFLEDForConditionalGeneration,) if is_tf_available() else ()
		__snake_case									=   (
		    {
		        'conversational': TFLEDForConditionalGeneration,
		        'feature-extraction': TFLEDModel,
		        'summarization': TFLEDForConditionalGeneration,
		        'text2text-generation': TFLEDForConditionalGeneration,
		        'translation': TFLEDForConditionalGeneration,
		    }
		    if is_tf_available()
		    else {}
		)
		__snake_case									=   True
		__snake_case									=   False
		__snake_case									=   False
		__snake_case									=   False



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  TFLEDModelTester(self   )
								lowerCAmelCase_      			=  ConfigTester(self,	config_class=UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								self.config_tester.run_common_tests()



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_			,					lowerCAmelCase_      			=  self.model_tester.prepare_config_and_inputs_for_common()
								lowerCAmelCase_      			=  tf.zeros_like(inputs_dict['''attention_mask''']   )
								lowerCAmelCase_      			=  2
								lowerCAmelCase_      			=  tf.where(
								    tf.range(self.model_tester.seq_length   )[None, :] < num_global_attn_indices,	1,	inputs_dict['''global_attention_mask'''],	)

								lowerCAmelCase_      			=  True
								lowerCAmelCase_      			=  self.model_tester.seq_length
								lowerCAmelCase_      			=  self.model_tester.encoder_seq_length

								def check_decoder_attentions_output(UpperCamelCase__   ):
														lowerCAmelCase_      			=  outputs.decoder_attentions
														self.assertEqual(len(UpperCamelCase__   ),	self.model_tester.num_hidden_layers   )
														self.assertListEqual(
														    list(decoder_attentions[0].shape[-3:]   ),	[self.model_tester.num_attention_heads, seq_length, seq_length],	)

								def check_encoder_attentions_output(UpperCamelCase__   ):
														lowerCAmelCase_      			=  [t.numpy() for t in outputs.encoder_attentions]
														lowerCAmelCase_      			=  [t.numpy() for t in outputs.encoder_global_attentions]
														self.assertEqual(len(UpperCamelCase__   ),	self.model_tester.num_hidden_layers   )
														self.assertEqual(len(UpperCamelCase__   ),	self.model_tester.num_hidden_layers   )
														self.assertListEqual(
														    list(attentions[0].shape[-3:]   ),	[self.model_tester.num_attention_heads, seq_length, seq_length],	)
														self.assertListEqual(
														    list(global_attentions[0].shape[-3:]   ),	[self.model_tester.num_attention_heads, encoder_seq_length, num_global_attn_indices],	)

								for model_class in self.all_model_classes:
														lowerCAmelCase_      			=  True
														lowerCAmelCase_      			=  False
														lowerCAmelCase_      			=  False
														lowerCAmelCase_      			=  model_class(UpperCamelCase__   )
														lowerCAmelCase_      			=  model(self._prepare_for_class(UpperCamelCase__,	UpperCamelCase__   )   )
														lowerCAmelCase_      			=  len(UpperCamelCase__   )
														self.assertEqual(config.output_hidden_states,	UpperCamelCase__   )
														check_encoder_attentions_output(UpperCamelCase__   )

														if self.is_encoder_decoder:
																				lowerCAmelCase_      			=  model_class(UpperCamelCase__   )
																				lowerCAmelCase_      			=  model(self._prepare_for_class(UpperCamelCase__,	UpperCamelCase__   )   )
																				self.assertEqual(config.output_hidden_states,	UpperCamelCase__   )
																				check_decoder_attentions_output(UpperCamelCase__   )

														# Check that output attentions can also be changed via the config
														del inputs_dict["output_attentions"]
														lowerCAmelCase_      			=  True
														lowerCAmelCase_      			=  model_class(UpperCamelCase__   )
														lowerCAmelCase_      			=  model(self._prepare_for_class(UpperCamelCase__,	UpperCamelCase__   )   )
														self.assertEqual(config.output_hidden_states,	UpperCamelCase__   )
														check_encoder_attentions_output(UpperCamelCase__   )

														# Check attention is always last and order is fine
														lowerCAmelCase_      			=  True
														lowerCAmelCase_      			=  True
														lowerCAmelCase_      			=  model_class(UpperCamelCase__   )
														lowerCAmelCase_      			=  model(self._prepare_for_class(UpperCamelCase__,	UpperCamelCase__   )   )

														self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1),	len(UpperCamelCase__   )   )
														self.assertEqual(model.config.output_hidden_states,	UpperCamelCase__   )
														check_encoder_attentions_output(UpperCamelCase__   )



		@unittest.skip('''LED keeps using potentially symbolic tensors in conditionals and breaks tracing.'''   )
		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								pass







		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								pass







def __UpperCamelCase			(       _A  ):
						return tf.constant(_A ,  dtype=tf.intaa  )


_A     =			1e-4






@slow
@require_tf
class  A		(  unittest.TestCase     ):



		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  TFLEDForConditionalGeneration.from_pretrained('''allenai/led-base-16384'''   ).led

								# change to intended input here
								lowerCAmelCase_      			=  _long_tensor([512 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]   )
								lowerCAmelCase_      			=  _long_tensor([128 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]   )
								lowerCAmelCase_      			=  prepare_led_inputs_dict(model.config,	UpperCamelCase__,	UpperCamelCase__   )
								lowerCAmelCase_      			=  model(**UpperCamelCase__   )[0]
								lowerCAmelCase_      			=  (1, 1024, 768)
								self.assertEqual(output.shape,	UpperCamelCase__   )
								# change to expected output here
								lowerCAmelCase_      			=  tf.convert_to_tensor(
								    [[2.3_050, 2.8_279, 0.6_531], [-1.8_457, -0.1_455, -3.5_661], [-1.0_186, 0.4_586, -2.2_043]],	)
								tf.debugging.assert_near(output[:, :3, :3],	UpperCamelCase__,	atol=1E-3   )







		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  TFLEDForConditionalGeneration.from_pretrained('''allenai/led-base-16384'''   )

								# change to intended input here
								lowerCAmelCase_      			=  _long_tensor([512 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]   )
								lowerCAmelCase_      			=  _long_tensor([128 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]   )
								lowerCAmelCase_      			=  prepare_led_inputs_dict(model.config,	UpperCamelCase__,	UpperCamelCase__   )
								lowerCAmelCase_      			=  model(**UpperCamelCase__   )[0]
								lowerCAmelCase_      			=  (1, 1024, model.config.vocab_size)
								self.assertEqual(output.shape,	UpperCamelCase__   )
								# change to expected output here
								lowerCAmelCase_      			=  tf.convert_to_tensor(
								    [[33.6_507, 6.4_572, 16.8_089], [5.8_739, -2.4_238, 11.2_902], [-3.2_139, -4.3_149, 4.2_783]],	)
								tf.debugging.assert_near(output[:, :3, :3],	UpperCamelCase__,	atol=1E-3,	rtol=1E-3   )










from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_A     =			logging.get_logger(__name__)


_A     =			{
    '''nielsr/canine-s''': 2_048,
}

# Unicode defines 1,114,112 total “codepoints”
_A     =			1_114_112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_A     =			0

_A     =			0xe0_00
_A     =			0xe0_01
_A     =			0xe0_02
_A     =			0xe0_03
_A     =			0xe0_04

# Maps special codepoints to human-readable names.
_A     =			{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_A     =			{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}


class  A		(  __UpperCAmelCase     ):
		__snake_case									=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(     self,	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=False,	UpperCamelCase__=2048,	**UpperCamelCase__,	):







								"""simple docstring"""



								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else bos_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else eos_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else sep_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else cls_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else mask_token

								super().__init__(
								    bos_token=UpperCamelCase__,	eos_token=UpperCamelCase__,	sep_token=UpperCamelCase__,	cls_token=UpperCamelCase__,	pad_token=UpperCamelCase__,	mask_token=UpperCamelCase__,	add_prefix_space=UpperCamelCase__,	model_max_length=UpperCamelCase__,	**UpperCamelCase__,	)

								# Creates a mapping for looking up the IDs of special symbols.
								lowerCAmelCase_      			=  {}
								for codepoint, name in SPECIAL_CODEPOINTS.items():
														lowerCAmelCase_      			=  codepoint

								# Creates a mapping for looking up the string forms of special symbol IDs.
								lowerCAmelCase_      			=  {
								    codepoint: name for name, codepoint in self._special_codepoints.items()
								}

								lowerCAmelCase_      			=  UNICODE_VOCAB_SIZE
								lowerCAmelCase_      			=  len(self._special_codepoints   )



		@property
		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								return self._unicode_vocab_size



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								return list(UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								try:
														return ord(UpperCamelCase__   )
								except TypeError:
														raise ValueError(f"invalid token: '{token}'"   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								try:
														if index in SPECIAL_CODEPOINTS:
																				return SPECIAL_CODEPOINTS[index]
														return chr(UpperCamelCase__   )
								except TypeError:
														raise ValueError(f"invalid id: {index}"   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								return "".join(UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  [self.sep_token_id]
								lowerCAmelCase_      			=  [self.cls_token_id]

								lowerCAmelCase_      			=  cls + token_ids_a + sep
								if token_ids_a is not None:
														result += token_ids_a + sep
								return result



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None,	UpperCamelCase__ = False   ):







								"""simple docstring"""



								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=UpperCamelCase__,	token_ids_a=UpperCamelCase__,	already_has_special_tokens=UpperCamelCase__   )

								lowerCAmelCase_      			=  [1] + ([0] * len(UpperCamelCase__   )) + [1]
								if token_ids_a is not None:
														result += ([0] * len(UpperCamelCase__   )) + [1]
								return result



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  [self.sep_token_id]
								lowerCAmelCase_      			=  [self.cls_token_id]

								lowerCAmelCase_      			=  len(cls + token_ids_a + sep   ) * [0]
								if token_ids_a is not None:
														result += len(token_ids_a + sep   ) * [1]
								return result







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								return ()




