

from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel





def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__=None   ,							UpperCamelCase__=None      ):







							'''simple docstring'''






							if attention_mask is None:
														snake_case_							=    tf.cast(tf.math.not_equal(UpperCamelCase__   ,							config.pad_token_id      )   ,							tf.inta      )
							return {"input_ids": input_ids, "attention_mask": attention_mask}







@require_tf
class  lowercase				:
				__SCREAMING_SNAKE_CASE					: Dict       			=			OPTConfig
				__SCREAMING_SNAKE_CASE					: int       			=			{}
				__SCREAMING_SNAKE_CASE					: Dict       			=			'''gelu'''




				def __init__(					self      ,   snake_case      ,   snake_case=13      ,   snake_case=7      ,   snake_case=True      ,   snake_case=False      ,   snake_case=99      ,   snake_case=16      ,   snake_case=2      ,   snake_case=4      ,   snake_case=4      ,   snake_case="gelu"      ,   snake_case=0.1      ,   snake_case=0.1      ,   snake_case=20      ,   snake_case=2      ,   snake_case=1      ,   snake_case=0      ,   snake_case=16      ,   snake_case=16      ,   ):
											snake_case_							=    parent
											snake_case_							=    batch_size
											snake_case_							=    seq_length
											snake_case_							=    is_training
											snake_case_							=    use_labels
											snake_case_							=    vocab_size
											snake_case_							=    hidden_size
											snake_case_							=    num_hidden_layers
											snake_case_							=    num_attention_heads
											snake_case_							=    intermediate_size
											snake_case_							=    hidden_act
											snake_case_							=    hidden_dropout_prob
											snake_case_							=    attention_probs_dropout_prob
											snake_case_							=    max_position_embeddings
											snake_case_							=    eos_token_id
											snake_case_							=    pad_token_id
											snake_case_							=    bos_token_id
											snake_case_							=    embed_dim
											snake_case_							=    word_embed_proj_dim
											snake_case_							=    False




				def     a							(					self       ):
											snake_case_							=    ids_tensor([self.batch_size, self.seq_length - 1]      ,   self.vocab_size       )
											snake_case_							=    tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )      ,   1       )
											snake_case_							=    tf.concat([input_ids, eos_tensor]      ,   axis=1       )

											snake_case_							=    self.config_cls(
											    vocab_size=self.vocab_size      ,   hidden_size=self.hidden_size      ,   num_hidden_layers=self.num_hidden_layers      ,   num_attention_heads=self.num_attention_heads      ,   ffn_dim=self.intermediate_size      ,   dropout=self.hidden_dropout_prob      ,   attention_dropout=self.attention_probs_dropout_prob      ,   max_position_embeddings=self.max_position_embeddings      ,   eos_token_id=self.eos_token_id      ,   bos_token_id=self.bos_token_id      ,   pad_token_id=self.pad_token_id      ,   embed_dim=self.embed_dim      ,   word_embed_proj_dim=self.word_embed_proj_dim      ,   is_encoder_decoder=snake_case      ,   **self.config_updates      ,   )
											snake_case_							=    prepare_opt_inputs_dict(snake_case      ,   snake_case       )
											return config, inputs_dict




				def     a							(					self      ,   snake_case      ,   snake_case       ):
											snake_case_							=    TFOPTModel(config=snake_case       )
											snake_case_							=    inputs_dict['input_ids']

											snake_case_							=    input_ids[:1, :]
											snake_case_							=    inputs_dict['attention_mask'][:1, :]
											snake_case_							=    1

											# first forward pass
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   use_cache=snake_case       )

											snake_case_				,    snake_case_							=    outputs.to_tuple()

											# create hypothetical next token and extent to next_input_ids
											snake_case_							=    ids_tensor((self.batch_size, 3)      ,   config.vocab_size       )
											snake_case_							=    tf.cast(ids_tensor((self.batch_size, 3)      ,   2       )      ,   tf.inta       )

											# append to next input_ids and
											snake_case_							=    tf.concat([input_ids, next_tokens]      ,   axis=-1       )
											snake_case_							=    tf.concat([attention_mask, next_attn_mask]      ,   axis=-1       )

											snake_case_							=    model(snake_case      ,   attention_mask=snake_case       )[0]
											snake_case_							=    model(snake_case      ,   attention_mask=snake_case      ,   past_key_values=snake_case       )[0]

											self.parent.assertEqual(next_tokens.shape[1]      ,   output_from_past.shape[1]       )

											# select random slice
											snake_case_							=    int(ids_tensor((1,)      ,   output_from_past.shape[-1]       )       )
											snake_case_							=    output_from_no_past[:, -3:, random_slice_idx]
											snake_case_							=    output_from_past[:, :, random_slice_idx]

											# test that outputs are equal for slice
											tf.debugging.assert_near(snake_case      ,   snake_case      ,   rtol=1e-3       )







@require_tf
class  lowercase				(       lowercase_	,  lowercase_	,  unittest.TestCase			):
				__SCREAMING_SNAKE_CASE					: Dict       			=			(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
				__SCREAMING_SNAKE_CASE					: List[str]       			=			(TFOPTForCausalLM,) if is_tf_available() else ()
				__SCREAMING_SNAKE_CASE					: Dict       			=			(
				    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
				)
				__SCREAMING_SNAKE_CASE					: int       			=			False
				__SCREAMING_SNAKE_CASE					: Union[str, Any]       			=			False
				__SCREAMING_SNAKE_CASE					: List[str]       			=			False
				__SCREAMING_SNAKE_CASE					: Union[str, Any]       			=			10




				def     a							(					self       ):
											snake_case_							=    TFOPTModelTester(self       )
											snake_case_							=    ConfigTester(self      ,   config_class=snake_case       )




				def     a							(					self       ):
											self.config_tester.run_common_tests()




				def     a							(					self       ):
											snake_case_							=    self.model_tester.prepare_config_and_inputs_for_common()
											self.model_tester.check_decoder_model_past_large_inputs(*snake_case       )




				def     a							(					self       ):
											snake_case_				,    snake_case_							=    self.model_tester.prepare_config_and_inputs_for_common()

											def _get_word_embedding_weight(snake_case      ,   snake_case       ):
																		if hasattr(snake_case      ,   'weight'       ):
																									return embedding_layer.weight
																		else:
																									# Here we build the word embeddings weights if not exists.
																									# And then we retry to get the attribute once built.
																									model.build()
																									if hasattr(snake_case      ,   'weight'       ):
																																return embedding_layer.weight
																									else:
																																return None

											for model_class in self.all_model_classes:
																		for size in [config.vocab_size - 10, config.vocab_size + 10]:
																									# build the embeddings
																									snake_case_							=    model_class(config=snake_case       )
																									snake_case_							=    _get_word_embedding_weight(snake_case      ,   model.get_input_embeddings()       )
																									snake_case_							=    _get_word_embedding_weight(snake_case      ,   model.get_output_embeddings()       )

																									# reshape the embeddings
																									model.resize_token_embeddings(snake_case       )
																									snake_case_							=    _get_word_embedding_weight(snake_case      ,   model.get_input_embeddings()       )
																									snake_case_							=    _get_word_embedding_weight(snake_case      ,   model.get_output_embeddings()       )

																									# check that the resized embeddings size matches the desired size.
																									snake_case_							=    size if size is not None else config.vocab_size

																									self.assertEqual(new_input_embeddings.shape[0]      ,   snake_case       )

																									# check that weights remain the same after resizing
																									snake_case_							=    True
																									for pa, pa in zip(old_input_embeddings.value()      ,   new_input_embeddings.value()       ):
																																if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																																							snake_case_							=    False
																									self.assertTrue(snake_case       )

																									if old_output_embeddings is not None and new_output_embeddings is not None:
																																self.assertEqual(new_output_embeddings.shape[0]      ,   snake_case       )

																																snake_case_							=    True
																																for pa, pa in zip(old_output_embeddings.value()      ,   new_output_embeddings.value()       ):
																																							if tf.math.reduce_sum(tf.math.abs(pa - pa       )       ) > 0:
																																														snake_case_							=    False
																																self.assertTrue(snake_case       )





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							return tf.constant(UpperCamelCase__   ,							dtype=tf.intaa      )







@require_tf
class  lowercase				(       unittest.TestCase			):
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			99




				def     a							(					self       ):
											snake_case_							=    tf.ones((4, 1)      ,   dtype=tf.intaa       ) * 2
											snake_case_							=    tf.concat([ids_tensor((4, 6)      ,   self.vocab_size - 3       ) + 3, eos_column_vector]      ,   axis=1       )
											snake_case_							=    input_ids.shape[0]
											snake_case_							=    OPTConfig(
											    vocab_size=self.vocab_size      ,   hidden_size=24      ,   num_hidden_layers=2      ,   num_attention_heads=2      ,   ffn_dim=32      ,   max_position_embeddings=48      ,   eos_token_id=2      ,   pad_token_id=1      ,   bos_token_id=0      ,   )
											return config, input_ids, batch_size







@require_sentencepiece
@require_tf
class  lowercase				(       unittest.TestCase			):




				@slow
				def     a							(					self       ):
											snake_case_							=    TFOPTModel.from_pretrained('facebook/opt-350m'       )
											snake_case_							=    _long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]]       )
											snake_case_							=    tf.not_equal(snake_case      ,   model.config.pad_token_id       )
											with tf.GradientTape():
																		snake_case_							=    model(input_ids=snake_case      ,   attention_mask=snake_case       ).last_hidden_state
											snake_case_							=    (1, 11, 512)
											self.assertEqual(output.shape      ,   snake_case       )
											snake_case_							=    tf.constant(
											    [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]       )
											self.assertTrue(np.allclose(output[:, :3, :3]      ,   snake_case      ,   atol=4e-3       )       )

											snake_case_							=    tf.function(snake_case      ,   jit_compile=snake_case       )
											snake_case_							=    xla_generate(snake_case      ,   snake_case       )[0]
											self.assertTrue(np.allclose(output[:, :3, :3]      ,   snake_case      ,   atol=4e-2       )       )







@require_tf
@slow
class  lowercase				(       unittest.TestCase			):




				def     a							(					self       ):
											super().setUp()
											snake_case_							=    'facebook/opt-350m'




				def     a							(					self       ):
											snake_case_							=    TFOPTForCausalLM.from_pretrained(self.path_model       )
											snake_case_							=    GPTaTokenizer.from_pretrained(self.path_model       )

											snake_case_							=    [
											    'Today is a beautiful day and I want to',
											    'In the city of',
											    'Paris is the capital of France and',
											    'Computers and mobile phones have taken',
											]
											# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
											snake_case_							=    tokenizer(snake_case      ,   return_tensors='tf'      ,   padding=snake_case      ,   add_special_tokens=snake_case       )
											snake_case_							=    tf.math.reduce_mean(model(inputs.input_ids      ,   attention_mask=inputs.attention_mask       )[0]      ,   axis=-1       )
											snake_case_							=    tf.constant(
											    [
											        [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
											        [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
											        [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
											        [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
											    ]       )
											self.assertTrue(np.allclose(snake_case      ,   snake_case      ,   atol=1e-4       )       )

											snake_case_							=    tf.function(snake_case      ,   jit_compile=snake_case       )
											snake_case_							=    tf.math.reduce_mean(xla_generate(inputs.input_ids      ,   attention_mask=inputs.attention_mask       )[0]      ,   axis=-1       )
											self.assertTrue(np.allclose(snake_case      ,   snake_case      ,   atol=1e-4       )       )







@require_tf
@slow
class  lowercase				(       unittest.TestCase			):




				@property
				def     a							(					self       ):
											return [
											    "Today is a beautiful day and I want",
											    "In the city of",
											    "Paris is the capital of France and",
											    "Computers and mobile phones have taken",
											]




				def     a							(					self       ):
											snake_case_							=    'facebook/opt-125m'

											snake_case_							=    [
											    'Today is a beautiful day and I want to',
											    'In the city of New York, the city',
											    'Paris is the capital of France and the capital',
											    'Computers and mobile phones have taken over the',
											]

											snake_case_							=    []
											snake_case_							=    GPTaTokenizer.from_pretrained(snake_case       )
											snake_case_							=    TFOPTForCausalLM.from_pretrained(snake_case       )

											for prompt in self.prompts:
																		snake_case_							=    tokenizer(snake_case      ,   return_tensors='tf'       ).input_ids

																		snake_case_							=    model.generate(snake_case      ,   max_length=10       )

																		snake_case_							=    tokenizer.batch_decode(snake_case      ,   skip_special_tokens=snake_case       )
																		predicted_outputs += generated_string

											self.assertListEqual(snake_case      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    'facebook/opt-350m'

											snake_case_							=    GPTaTokenizer.from_pretrained(snake_case       )
											snake_case_							=    TFOPTForCausalLM.from_pretrained(snake_case       )

											snake_case_							=    'left'

											# use different length sentences to test batching
											snake_case_							=    [
											    'Hello, my dog is a little',
											    'Today, I',
											]

											snake_case_							=    tokenizer(snake_case      ,   return_tensors='tf'      ,   padding=snake_case       )
											snake_case_							=    inputs['input_ids']

											snake_case_							=    model.generate(input_ids=snake_case      ,   attention_mask=inputs['attention_mask']       )

											snake_case_							=    tokenizer(sentences[0]      ,   return_tensors='tf'       ).input_ids
											snake_case_							=    model.generate(input_ids=snake_case       )

											snake_case_							=    inputs_non_padded.shape[-1] - tf.math.reduce_sum(
											    tf.cast(inputs['attention_mask'][-1]      ,   tf.intaa       )       )
											snake_case_							=    tokenizer(sentences[1]      ,   return_tensors='tf'       ).input_ids
											snake_case_							=    model.generate(input_ids=snake_case      ,   max_length=model.config.max_length - num_paddings       )

											snake_case_							=    tokenizer.batch_decode(snake_case      ,   skip_special_tokens=snake_case       )
											snake_case_							=    tokenizer.decode(output_non_padded[0]      ,   skip_special_tokens=snake_case       )
											snake_case_							=    tokenizer.decode(output_padded[0]      ,   skip_special_tokens=snake_case       )

											snake_case_							=    [
											    'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
											    'Today, I was in the middle of a conversation with a friend about the',
											]
											self.assertListEqual(snake_case      ,   snake_case       )
											self.assertListEqual(snake_case      ,   [non_padded_sentence, padded_sentence]       )




				def     a							(					self       ):
											snake_case_							=    'facebook/opt-350m'

											snake_case_							=    [
											    'Today is a beautiful day and I want to',
											    'In the city of San Francisco, the city',
											    'Paris is the capital of France and the capital',
											    'Computers and mobile phones have taken over the',
											]

											snake_case_							=    []
											snake_case_							=    GPTaTokenizer.from_pretrained(snake_case       )
											snake_case_							=    TFOPTForCausalLM.from_pretrained(snake_case       )

											for prompt in self.prompts:
																		snake_case_							=    tokenizer(snake_case      ,   return_tensors='tf'       ).input_ids

																		snake_case_							=    model.generate(snake_case      ,   max_length=10       )

																		snake_case_							=    tokenizer.batch_decode(snake_case      ,   skip_special_tokens=snake_case       )
																		predicted_outputs += generated_string

											self.assertListEqual(snake_case      ,   snake_case       )







import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    'files'   ,							[
        ['full:README.md', 'dataset_infos.json'],
        ['empty:README.md', 'dataset_infos.json'],
        ['dataset_infos.json'],
        ['full:README.md'],
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    tmp_path_factory.mktemp('dset_infos_dir'      )
							if "full:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write('---\ndataset_info:\n  dataset_size: 42\n---'      )
							if "empty:README.md" in files:
														with open(dataset_infos_dir / 'README.md'   ,							'w'      ) as f:
																					f.write(''      )
    # we want to support dataset_infos.json for backward compatibility
							if "dataset_infos.json" in files:
														with open(dataset_infos_dir / 'dataset_infos.json'   ,							'w'      ) as f:
																					f.write('{"default": {"dataset_size": 42}}'      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )
							assert dataset_infos
							assert dataset_infos["default"].dataset_size == 42





@pytest.mark.parametrize(
    'dataset_info'   ,							[
        DatasetInfo(),
        DatasetInfo(
            description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_info.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfo.from_directory(UpperCamelCase__      )
							assert dataset_info == reloaded
							assert os.path.exists(os.path.join(UpperCamelCase__   ,							'dataset_info.json'      )      )





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo(
							    description='foo'   ,							citation='bar'   ,							homepage='https://foo.bar'   ,							license='CC0'   ,							features=Features({'a': Value('int32'      )}      )   ,							post_processed={}   ,							supervised_keys=()   ,							task_templates=[]   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train', 'num_examples': 42}]   ,							download_checksums={}   ,							download_size=1337   ,							post_processing_size=442   ,							dataset_size=1234   ,							size_in_bytes=1337 + 442 + 1234   ,							)
							snake_case_							=    dataset_info._to_yaml_dict()
							assert sorted(UpperCamelCase__      ) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML      )
							for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
														assert key in dataset_info_yaml_dict
														assert isinstance(dataset_info_yaml_dict[key]   ,							(list, dict, int, str)      )
							snake_case_							=    yaml.safe_dump(UpperCamelCase__      )
							snake_case_							=    yaml.safe_load(UpperCamelCase__      )
							assert dataset_info_yaml_dict == reloaded





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    DatasetInfo()
							snake_case_							=    dataset_info._to_yaml_dict()
							assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    'dataset_infos_dict'   ,							[
        DatasetInfosDict(),
        DatasetInfosDict({'default': DatasetInfo()}      ),
        DatasetInfosDict({'my_config_name': DatasetInfo()}      ),
        DatasetInfosDict(
            {
                'default': DatasetInfo(
                    description='foo'   ,							features=Features({'a': Value('int32'      )}      )   ,							builder_name='builder'   ,							config_name='config'   ,							version='1.0.0'   ,							splits=[{'name': 'train'}]   ,							download_size=42   ,							)
            }      ),
        DatasetInfosDict(
            {
                'v1': DatasetInfo(dataset_size=42      ),
                'v2': DatasetInfo(dataset_size=1337      ),
            }      ),
    ]   ,							)
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    str(UpperCamelCase__      )
							dataset_infos_dict.write_to_directory(UpperCamelCase__      )
							snake_case_							=    DatasetInfosDict.from_directory(UpperCamelCase__      )

							# the config_name of the dataset_infos_dict take over the attribute
							for config_name, dataset_info in dataset_infos_dict.items():
														snake_case_							=    config_name
														# the yaml representation doesn't include fields like description or citation
														# so we just test that we can recover what we can from the yaml
														snake_case_							=    DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()      )
							assert dataset_infos_dict == reloaded

							if dataset_infos_dict:
														assert os.path.exists(os.path.join(UpperCamelCase__   ,							'README.md'      )      )




