


'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel
@require_tf
class   __lowerCAmelCase						:





      '''simple docstring'''






      lowerCAmelCase__					:     Optional[int]  					=							MBartConfig
      lowerCAmelCase__					:     Optional[int]  					=							{}
      lowerCAmelCase__					:     Optional[int]  					=							"""gelu"""



      def __init__(self    :    Tuple   ,						UpperCamelCase    :    Any   ,						UpperCamelCase    :    int=13   ,						UpperCamelCase    :    List[str]=7   ,						UpperCamelCase    :    Dict=True   ,						UpperCamelCase    :    List[Any]=False   ,						UpperCamelCase    :    Dict=99   ,						UpperCamelCase    :    Dict=32   ,						UpperCamelCase    :    Tuple=2   ,						UpperCamelCase    :    Any=4   ,						UpperCamelCase    :    Dict=37   ,						UpperCamelCase    :    Union[str, Any]=0.1   ,						UpperCamelCase    :    Union[str, Any]=0.1   ,						UpperCamelCase    :    Dict=20   ,						UpperCamelCase    :    Optional[int]=2   ,						UpperCamelCase    :    Optional[Any]=1   ,						UpperCamelCase    :    str=0   ,						):

             '''simple docstring'''







             lowercase__      			= parent
             lowercase__      			= batch_size
             lowercase__      			= seq_length
             lowercase__      			= is_training
             lowercase__      			= use_labels
             lowercase__      			= vocab_size
             lowercase__      			= hidden_size
             lowercase__      			= num_hidden_layers
             lowercase__      			= num_attention_heads
             lowercase__      			= intermediate_size
             lowercase__      			= hidden_dropout_prob
             lowercase__      			= attention_probs_dropout_prob
             lowercase__      			= max_position_embeddings
             lowercase__      			= eos_token_id
             lowercase__      			= pad_token_id
             lowercase__      			= bos_token_id



      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= ids_tensor([self.batch_size, self.seq_length - 1]   ,						self.vocab_size )
             lowercase__      			= tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size )   ,						1 )
             lowercase__      			= tf.concat([input_ids, eos_tensor]   ,						axis=1 )

             lowercase__      			= ids_tensor([self.batch_size, self.seq_length]   ,						self.vocab_size )

             lowercase__      			= self.config_cls(
                 vocab_size=self.vocab_size   ,						d_model=self.hidden_size   ,						encoder_layers=self.num_hidden_layers   ,						decoder_layers=self.num_hidden_layers   ,						encoder_attention_heads=self.num_attention_heads   ,						decoder_attention_heads=self.num_attention_heads   ,						encoder_ffn_dim=self.intermediate_size   ,						decoder_ffn_dim=self.intermediate_size   ,						dropout=self.hidden_dropout_prob   ,						attention_dropout=self.attention_probs_dropout_prob   ,						max_position_embeddings=self.max_position_embeddings   ,						eos_token_ids=[2]   ,						bos_token_id=self.bos_token_id   ,						pad_token_id=self.pad_token_id   ,						decoder_start_token_id=self.pad_token_id   ,						**self.config_updates   ,						)
             lowercase__      			= prepare_mbart_inputs_dict(UpperCamelCase   ,						UpperCamelCase   ,						UpperCamelCase )
             return config, inputs_dict






      def       UpperCamelCase__		(self    :    Tuple   ,						UpperCamelCase    :    List[str]   ,						UpperCamelCase    :    List[Any] ):

             '''simple docstring'''







             lowercase__      			= TFMBartModel(config=UpperCamelCase ).get_decoder()
             lowercase__      			= inputs_dict['''input_ids''']

             lowercase__      			= input_ids[:1, :]
             lowercase__      			= inputs_dict['''attention_mask'''][:1, :]
             lowercase__      			= inputs_dict['''head_mask''']
             lowercase__      			= 1

             # first forward pass
             lowercase__      			= model(UpperCamelCase   ,						attention_mask=UpperCamelCase   ,						head_mask=UpperCamelCase   ,						use_cache=UpperCamelCase )

             lowercase__			,lowercase__      			= outputs.to_tuple()
             lowercase__      			= past_key_values[1]






def _SCREAMING_SNAKE_CASE      (A  , A  , A  , A=None  , A=None  , A=None  , A=None  , A=None  , )       ->       List[Any]:






       """simple docstring"""







       if attention_mask is None:
              lowercase__      			= tf.cast(tf.math.not_equal(A  , config.pad_token_id       )  , tf.inta       )
       if decoder_attention_mask is None:
              lowercase__      			= tf.concat(
                  [
                      tf.ones(decoder_input_ids[:, :1].shape  , dtype=tf.inta       ),
                      tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]  , config.pad_token_id       )  , tf.inta       ),
                  ]  , axis=-1  , )
       if head_mask is None:
              lowercase__      			= tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
       if decoder_head_mask is None:
              lowercase__      			= tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
       if cross_attn_head_mask is None:
              lowercase__      			= tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
       return {
           "input_ids": input_ids,
           "decoder_input_ids": decoder_input_ids,
           "attention_mask": attention_mask,
           "decoder_attention_mask": decoder_attention_mask,
           "head_mask": head_mask,
           "decoder_head_mask": decoder_head_mask,
           "cross_attn_head_mask": cross_attn_head_mask,
       }
@require_tf
class   __lowerCAmelCase						(lowercase_  ,     lowercase_  ,     unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Dict  					=							(TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
      lowerCAmelCase__					:     Optional[Any]  					=							(TFMBartForConditionalGeneration,) if is_tf_available() else ()
      lowerCAmelCase__					:     Optional[int]  					=							(
          {
              """conversational""": TFMBartForConditionalGeneration,
              """feature-extraction""": TFMBartModel,
              """summarization""": TFMBartForConditionalGeneration,
              """text2text-generation""": TFMBartForConditionalGeneration,
              """translation""": TFMBartForConditionalGeneration,
          }
          if is_tf_available()
          else {}
      )
      lowerCAmelCase__					:     Any  					=							True
      lowerCAmelCase__					:     str  					=							False
      lowerCAmelCase__					:     str  					=							False



      def       UpperCamelCase__		(self    :    Any   ,						UpperCamelCase    :    Dict   ,						UpperCamelCase    :    Tuple   ,						UpperCamelCase    :    Optional[Any]   ,						UpperCamelCase    :    Union[str, Any]   ,						UpperCamelCase    :    Dict ):

             '''simple docstring'''







             if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
                    # Exception encountered when calling layer '...'
                    return True

             return False



      def       UpperCamelCase__		(self    :    int ):

             '''simple docstring'''







             lowercase__      			= TFMBartModelTester(self )
             lowercase__      			= ConfigTester(self   ,						config_class=UpperCamelCase )



      def       UpperCamelCase__		(self    :    List[str] ):

             '''simple docstring'''







             self.config_tester.run_common_tests()






      def       UpperCamelCase__		(self    :    Tuple ):

             '''simple docstring'''







             lowercase__      			= self.model_tester.prepare_config_and_inputs_for_common()
             self.model_tester.check_decoder_model_past_large_inputs(*UpperCamelCase )






@require_sentencepiece
@require_tokenizers
@require_tf
class   __lowerCAmelCase						(unittest.TestCase     ):





      '''simple docstring'''






      lowerCAmelCase__					:     Optional[int]  					=							[
          """ UN Chief Says There Is No Military Solution in Syria""",
      ]
      lowerCAmelCase__					:     str  					=							[
          """Şeful ONU declară că nu există o soluţie militară în Siria""",
      ]
      lowerCAmelCase__					:     str  					=							"""facebook/mbart-large-en-ro"""



      @cached_property
      def       UpperCamelCase__		(self    :    Optional[int] ):

             '''simple docstring'''







             return AutoTokenizer.from_pretrained(self.model_name )



      @cached_property
      def       UpperCamelCase__		(self    :    Optional[Any] ):

             '''simple docstring'''







             lowercase__      			= TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name )
             return model



      def       UpperCamelCase__		(self    :    Union[str, Any]   ,						**UpperCamelCase    :    Optional[int] ):

             '''simple docstring'''







             lowercase__      			= self.translate_src_text(**UpperCamelCase )
             self.assertListEqual(self.expected_text   ,						UpperCamelCase )



      def       UpperCamelCase__		(self    :    Tuple   ,						**UpperCamelCase    :    Dict ):

             '''simple docstring'''







             lowercase__      			= self.tokenizer(self.src_text   ,						**UpperCamelCase   ,						return_tensors='''tf''' )
             lowercase__      			= self.model.generate(
                 model_inputs.input_ids   ,						attention_mask=model_inputs.attention_mask   ,						num_beams=2 )
             lowercase__      			= self.tokenizer.batch_decode(UpperCamelCase   ,						skip_special_tokens=UpperCamelCase )
             return generated_words






      @slow
      def       UpperCamelCase__		(self    :    Union[str, Any] ):

             '''simple docstring'''







             self._assert_generated_batch_equal_expected()







import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_UpperCamelCase 			=		logging.get_logger(__name__)
_UpperCamelCase 			=		{"vocab_file": "spiece.model"}

_UpperCamelCase 			=		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

_UpperCamelCase 			=		{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}







class   __lowercase      (_UpperCAmelCase ):
 _UpperCamelCase			  =  VOCAB_FILES_NAMES
 _UpperCamelCase			  =  PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase			  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase			  =  ["""input_ids""", """attention_mask"""]
 def __init__(							self      ,							A_      ,							A_=False      ,							A_=False      ,							A_=False      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_ = None      ,							**A_      ,							)					->None:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					{} if sp_model_kwargs is None else sp_model_kwargs

       __lowerCAmelCase					:						int					=					kwargs.get('''name_or_path'''					)
       if name_or_path is None:
             logger.warning(
                 '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                 ''' you are testing the model, this can safely be ignored'''					)
             __lowerCAmelCase					:						Union[str, Any]					=					'''None'''

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       __lowerCAmelCase					:						str					=					'''<|endoftext|>''' if eos_token is None else eos_token
       __lowerCAmelCase					:						Any					=					'''<unk>''' if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             __lowerCAmelCase					:						Dict					=					unk_token if pad_token is None else pad_token
             __lowerCAmelCase					:						int					=					eos_token if bos_token is None else bos_token
       else:
             __lowerCAmelCase					:						Optional[int]					=					'''<pad>''' if pad_token is None else pad_token
             __lowerCAmelCase					:						List[str]					=					'''<s>''' if bos_token is None else bos_token

       super().__init__(
           do_lower_case=A_      ,							remove_space=A_      ,							keep_accents=A_      ,							bos_token=A_      ,							eos_token=A_      ,							unk_token=A_      ,							pad_token=A_      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**A_      ,							)

       __lowerCAmelCase					:						Union[str, Any]					=					do_lower_case
       __lowerCAmelCase					:						Union[str, Any]					=					remove_space
       __lowerCAmelCase					:						int					=					keep_accents
       __lowerCAmelCase					:						Union[str, Any]					=					vocab_file

       __lowerCAmelCase					:						int					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(A_					)

       # Used for whitespace normalization in input texts
       # fmt : off
       __lowerCAmelCase					:						List[Any]					=					{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       __lowerCAmelCase					:						int					=					re.compile(
           f"""[{"".join(map(A_      ,							list(range(0      ,							9					)					) + list(range(11      ,							32					)					) + list(range(127      ,							160					)					) + [160, 173, 8203]					)					)}]"""					)
 def __getstate__(							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Union[str, Any]					=					self.__dict__.copy()
       __lowerCAmelCase					:						List[Any]					=					None
       return state
 def __setstate__(							self      ,							A_					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					d

       # for backward compatibility
       if not hasattr(self      ,							'''sp_model_kwargs'''					):
             __lowerCAmelCase					:						List[Any]					=					{}

       __lowerCAmelCase					:						List[str]					=					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
       self.sp_model.Load(self.vocab_file					)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       return len(self.sp_model					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.non_printing_characters_re.sub(''''''      ,							A_					)

       # Normalize whitespaces
       __lowerCAmelCase					:						List[str]					=					''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]					)

       # NFC Unicode normalization
       __lowerCAmelCase					:						Tuple					=					unicodedata.normalize('''NFC'''      ,							A_					)
       return text
 def 	UpperCamelCase__  (							self      ,							A_      ,							**A_					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					self.preprocess_text(A_					)
       return self.sp_model.encode(A_      ,							out_type=A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->int:



       '''simple docstring'''





       return self.sp_model.PieceToId(A_					)
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.IdToPiece(A_					)
 @staticmethod
 def 	UpperCamelCase__  (							A_					)					->str:



       '''simple docstring'''





       return out_string
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					[]
       __lowerCAmelCase					:						Tuple					=					''''''
       __lowerCAmelCase					:						int					=					False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(A_					) + token
                   __lowerCAmelCase					:						Optional[Any]					=					True
                   __lowerCAmelCase					:						Optional[int]					=					[]
             else:
                   current_sub_tokens.append(A_					)
                   __lowerCAmelCase					:						str					=					False
       out_string += self.sp_model.decode(A_					)

       return out_string
 def 	UpperCamelCase__  (							self					)					->Dict[str, int]:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					{self.convert_ids_to_tokens(A_					): i for i in range(self.vocab_size					)}
       vocab.update(self.added_tokens_encoder					)
       return vocab
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = None					)					->Tuple[str]:



       '''simple docstring'''





       if not os.path.isdir(A_					):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
             return
       __lowerCAmelCase					:						Any					=					os.path.join(
           A_      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

       if os.path.abspath(self.vocab_file					) != os.path.abspath(A_					) and os.path.isfile(self.vocab_file					):
             copyfile(self.vocab_file      ,							A_					)
       elif not os.path.isfile(self.vocab_file					):
             with open(A_      ,							'''wb'''					) as fi:
                   __lowerCAmelCase					:						Dict					=					self.sp_model.serialized_model_proto()
                   fi.write(A_					)

       return (out_vocab_file,)
 def 	UpperCamelCase__  (							self      ,							A_      ,							A_ = False					)					->Union[List[int], List[List[int]], "torch.Tensor"]:



       '''simple docstring'''





       if isinstance(A_      ,							A_					):
             __lowerCAmelCase					:						Optional[Any]					=					self.preprocess_text(A_					)
             __lowerCAmelCase					:						Dict					=					self.sp_model.encode(A_					)
       else:
             __lowerCAmelCase					:						Dict					=					[self.preprocess_text(A_					) for t in text]
             __lowerCAmelCase					:						Optional[int]					=					self.sp_model.encode(A_					)

       if return_tensors is True or return_tensors == "pt":
             __lowerCAmelCase					:						Tuple					=					torch.tensor(A_					)

       return token_ids
 def 	UpperCamelCase__  (							self      ,							A_					)					->str:



       '''simple docstring'''





       return self.sp_model.decode(A_					)





 def 	UpperCamelCase__  (							self      ,							A_					)					->List[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
       __lowerCAmelCase					:						Any					=					(
           f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(A_					) + f"""{self.bos_token}Bot:"""
       )
       return self.encode(text=A_					)


