

'''simple docstring'''
import math
import unittest

from transformers import BioGptConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         BioGptForCausalLM,
         BioGptForSequenceClassification,
         BioGptForTokenClassification,
         BioGptModel,
         BioGptTokenizer,
     )
     from transformers.models.biogpt.modeling_biogpt import BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST







class 							A__    :





  def __init__(		self      :	str							,					_a      :	List[Any]							,					_a      :	Optional[Any]=13							,					_a      :	Tuple=7							,					_a      :	int=True							,					_a      :	Tuple=True							,					_a      :	Any=False							,					_a      :	Tuple=True							,					_a      :	Optional[int]=99							,					_a      :	List[Any]=32							,					_a      :	Dict=5							,					_a      :	List[str]=4							,					_a      :	str=37							,					_a      :	Dict="gelu"							,					_a      :	Optional[int]=0.1							,					_a      :	int=0.1							,					_a      :	List[Any]=512							,					_a      :	int=16							,					_a      :	List[str]=2							,					_a      :	Union[str, Any]=0.02							,					_a      :	Any=3							,					_a      :	str=4							,					_a      :	str=None							,					)	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =parent
       _SCREAMING_SNAKE_CASE							    =batch_size
       _SCREAMING_SNAKE_CASE							    =seq_length
       _SCREAMING_SNAKE_CASE							    =is_training
       _SCREAMING_SNAKE_CASE							    =use_input_mask
       _SCREAMING_SNAKE_CASE							    =use_token_type_ids
       _SCREAMING_SNAKE_CASE							    =use_labels
       _SCREAMING_SNAKE_CASE							    =vocab_size
       _SCREAMING_SNAKE_CASE							    =hidden_size
       _SCREAMING_SNAKE_CASE							    =num_hidden_layers
       _SCREAMING_SNAKE_CASE							    =num_attention_heads
       _SCREAMING_SNAKE_CASE							    =intermediate_size
       _SCREAMING_SNAKE_CASE							    =hidden_act
       _SCREAMING_SNAKE_CASE							    =hidden_dropout_prob
       _SCREAMING_SNAKE_CASE							    =attention_probs_dropout_prob
       _SCREAMING_SNAKE_CASE							    =max_position_embeddings
       _SCREAMING_SNAKE_CASE							    =type_vocab_size
       _SCREAMING_SNAKE_CASE							    =type_sequence_label_size
       _SCREAMING_SNAKE_CASE							    =initializer_range
       _SCREAMING_SNAKE_CASE							    =num_labels
       _SCREAMING_SNAKE_CASE							    =num_choices
       _SCREAMING_SNAKE_CASE							    =scope





  def 						A  (		self      :	Optional[Any] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size, self.seq_length]							,					self.vocab_size )

       _SCREAMING_SNAKE_CASE							    =None
       if self.use_input_mask:
            _SCREAMING_SNAKE_CASE							    =random_attention_mask([self.batch_size, self.seq_length] )

       _SCREAMING_SNAKE_CASE							    =None
       if self.use_token_type_ids:
            _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size, self.seq_length]							,					self.type_vocab_size )

       _SCREAMING_SNAKE_CASE							    =None
       _SCREAMING_SNAKE_CASE							    =None
       _SCREAMING_SNAKE_CASE							    =None
       if self.use_labels:
            _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size]							,					self.type_sequence_label_size )
            _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size, self.seq_length]							,					self.num_labels )
            _SCREAMING_SNAKE_CASE							    =ids_tensor([self.batch_size]							,					self.num_choices )

       _SCREAMING_SNAKE_CASE							    =self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





  def 						A  (		self      :	Optional[int] )	->    Any:





       '''simple docstring'''






       return BioGptConfig(
           vocab_size=self.vocab_size							,					hidden_size=self.hidden_size							,					num_hidden_layers=self.num_hidden_layers							,					num_attention_heads=self.num_attention_heads							,					intermediate_size=self.intermediate_size							,					hidden_act=self.hidden_act							,					hidden_dropout_prob=self.hidden_dropout_prob							,					attention_probs_dropout_prob=self.attention_probs_dropout_prob							,					max_position_embeddings=self.max_position_embeddings							,					type_vocab_size=self.type_vocab_size							,					is_decoder=_a							,					initializer_range=self.initializer_range							,					)





  def 						A  (		self      :	Tuple							,					_a      :	str							,					_a      :	str							,					_a      :	Optional[int]							,					_a      :	List[str]							,					_a      :	Dict							,					_a      :	Union[str, Any]							,					_a      :	int )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptModel(config=_a )
       model.to(_a )
       model.eval()
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a )
       _SCREAMING_SNAKE_CASE							    =model(_a )
       self.parent.assertEqual(result.last_hidden_state.shape							,					(self.batch_size, self.seq_length, self.hidden_size) )





  def 						A  (		self      :	Union[str, Any]							,					_a      :	List[str]							,					_a      :	Optional[Any]							,					_a      :	str							,					_a      :	Optional[int]							,					_a      :	Optional[Any]							,					_a      :	Dict							,					_a      :	Optional[int]							,					_a      :	List[Any]							,					_a      :	str							,					)	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptForCausalLM(config=_a )
       model.to(_a )
       model.eval()
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					token_type_ids=_a							,					labels=_a )
       self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.vocab_size) )





  def 						A  (		self      :	int							,					_a      :	Dict							,					_a      :	Tuple							,					_a      :	List[str]							,					_a      :	str							,					_a      :	Optional[int]							,					*_a      :	str )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptModel(config=_a )
       model.to(_a )
       model.eval()

       # create attention mask
       _SCREAMING_SNAKE_CASE							    =torch.ones(input_ids.shape							,					dtype=torch.long							,					device=_a )
       _SCREAMING_SNAKE_CASE							    =self.seq_length // 2
       _SCREAMING_SNAKE_CASE							    =0

       # first forward pass
       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a ).to_tuple()

       # create hypothetical next token and extent to next_input_ids
       _SCREAMING_SNAKE_CASE							    =ids_tensor((self.batch_size, 1)							,					config.vocab_size )

       # change a random masked slice from input_ids
       _SCREAMING_SNAKE_CASE							    =ids_tensor((1,)							,					_a ).item() + 1
       _SCREAMING_SNAKE_CASE							    =ids_tensor((self.batch_size, 1)							,					config.vocab_size ).squeeze(-1 )
       _SCREAMING_SNAKE_CASE							    =random_other_next_tokens

       # append to next input_ids and attn_mask
       _SCREAMING_SNAKE_CASE							    =torch.cat([input_ids, next_tokens]							,					dim=-1 )
       _SCREAMING_SNAKE_CASE							    =torch.cat(
           [attn_mask, torch.ones((attn_mask.shape[0], 1)							,					dtype=torch.long							,					device=_a )]							,					dim=1							,					)

       # get two different outputs
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a )['last_hidden_state']
       _SCREAMING_SNAKE_CASE							    =model(_a							,					past_key_values=_a							,					attention_mask=_a )['last_hidden_state']

       # select random slice
       _SCREAMING_SNAKE_CASE							    =ids_tensor((1,)							,					output_from_past.shape[-1] ).item()
       _SCREAMING_SNAKE_CASE							    =output_from_no_past[:, -1, random_slice_idx].detach()
       _SCREAMING_SNAKE_CASE							    =output_from_past[:, 0, random_slice_idx].detach()

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(_a							,					_a							,					atol=1e-3 ) )





  def 						A  (		self      :	Any							,					_a      :	Dict							,					_a      :	List[Any]							,					_a      :	str							,					_a      :	List[str]							,					_a      :	Any							,					*_a      :	List[str] )	->    str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptModel(config=_a ).to(_a ).eval()

       _SCREAMING_SNAKE_CASE							    =torch.ones(input_ids.shape							,					dtype=torch.long							,					device=_a )

       # first forward pass
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					use_cache=_a )

       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =outputs.to_tuple()

       # create hypothetical multiple next token and extent to next_input_ids
       _SCREAMING_SNAKE_CASE							    =ids_tensor((self.batch_size, 3)							,					config.vocab_size )
       _SCREAMING_SNAKE_CASE							    =ids_tensor((self.batch_size, 3)							,					2 )

       # append to next input_ids and
       _SCREAMING_SNAKE_CASE							    =torch.cat([input_ids, next_tokens]							,					dim=-1 )
       _SCREAMING_SNAKE_CASE							    =torch.cat([attention_mask, next_attn_mask]							,					dim=-1 )

       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a )['last_hidden_state']
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					past_key_values=_a )[
           'last_hidden_state'
       ]

       # select random slice
       _SCREAMING_SNAKE_CASE							    =ids_tensor((1,)							,					output_from_past.shape[-1] ).item()
       _SCREAMING_SNAKE_CASE							    =output_from_no_past[:, -3:, random_slice_idx].detach()
       _SCREAMING_SNAKE_CASE							    =output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1] )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(_a							,					_a							,					atol=1e-3 ) )





  def 						A  (		self      :	Optional[int]							,					_a      :	Dict							,					_a      :	Any							,					_a      :	List[Any]							,					_a      :	Tuple							,					_a      :	Dict							,					*_a      :	Optional[Any]							,					_a      :	int=False )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptForCausalLM(_a )
       model.to(_a )
       if gradient_checkpointing:
            model.gradient_checkpointing_enable()

       _SCREAMING_SNAKE_CASE							    =model(_a							,					labels=_a )
       self.parent.assertEqual(result.loss.shape							,					() )
       self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.vocab_size) )
       result.loss.backward()





  def 						A  (		self      :	int							,					_a      :	Dict							,					*_a      :	str )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptModel(_a )
       _SCREAMING_SNAKE_CASE							    =model.config.initializer_range / math.sqrt(2 * model.config.num_hidden_layers )
       for key in model.state_dict().keys():
            if "c_proj" in key and "weight" in key:
                 self.parent.assertLessEqual(abs(torch.std(model.state_dict()[key] ) - model_std )							,					0.0_01 )
                 self.parent.assertLessEqual(abs(torch.mean(model.state_dict()[key] ) - 0.0 )							,					0.01 )





  def 						A  (		self      :	Optional[int]							,					_a      :	Dict							,					_a      :	int							,					_a      :	Tuple							,					_a      :	Union[str, Any]							,					_a      :	Optional[int]							,					*_a      :	int )	->    str:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.num_labels
       _SCREAMING_SNAKE_CASE							    =BioGptForTokenClassification(_a )
       model.to(_a )
       model.eval()
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					token_type_ids=_a )
       self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.num_labels) )





  def 						A  (		self      :	int )	->    Optional[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.prepare_config_and_inputs()
       (
           (
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						(
           _SCREAMING_SNAKE_CASE
       )    ,						
       )							    =config_and_inputs
       _SCREAMING_SNAKE_CASE							    ={'input_ids': input_ids, 'attention_mask': input_mask}
       return config, inputs_dict







@require_torch
class 							A__    (       A__     ,   A__     ,   A__     ,   unittest.TestCase  ):
  A__		 =				(
      (BioGptModel, BioGptForCausalLM, BioGptForSequenceClassification, BioGptForTokenClassification)
      if is_torch_available()
      else ()
  )
  A__		 =				(BioGptForCausalLM,) if is_torch_available() else ()
  A__		 =				(
      {
          'feature-extraction': BioGptModel,
          'text-classification': BioGptForSequenceClassification,
          'text-generation': BioGptForCausalLM,
          'token-classification': BioGptForTokenClassification,
          'zero-shot': BioGptForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  A__		 =				False





  def 						A  (		self      :	Optional[int] )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptModelTester(self )
       _SCREAMING_SNAKE_CASE							    =ConfigTester(self							,					config_class=_a							,					hidden_size=37 )





  def 						A  (		self      :	str )	->    Tuple:





       '''simple docstring'''






       self.config_tester.run_common_tests()





  def 						A  (		self      :	Optional[int] )	->    List[Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_a )





  def 						A  (		self      :	Tuple )	->    Tuple:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
            _SCREAMING_SNAKE_CASE							    =type
            self.model_tester.create_and_check_model(*_a )





  def 						A  (		self      :	Dict )	->    int:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_biogpt_model_attention_mask_past(*_a )





  def 						A  (		self      :	List[str] )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_forward_and_backwards(*_a							,					gradient_checkpointing=_a )





  def 						A  (		self      :	Any )	->    Optional[int]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_biogpt_model_past_large_inputs(*_a )





  def 						A  (		self      :	List[str] )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_biogpt_weight_initialization(*_a )





  def 						A  (		self      :	Any )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_biogpt_for_token_classification(*_a )





  @slow
  def 						A  (		self      :	List[str] )	->    Dict:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptForCausalLM.from_pretrained('microsoft/biogpt' )
       model.to(_a )
       _SCREAMING_SNAKE_CASE							    =BioGptTokenizer.from_pretrained('microsoft/biogpt' )

       _SCREAMING_SNAKE_CASE							    ='left'

       # Define PAD Token = EOS Token = 50256
       _SCREAMING_SNAKE_CASE							    =tokenizer.eos_token
       _SCREAMING_SNAKE_CASE							    =model.config.eos_token_id

       # use different length sentences to test batching
       _SCREAMING_SNAKE_CASE							    =[
           'Hello, my dog is a little',
           'Today, I',
       ]

       _SCREAMING_SNAKE_CASE							    =tokenizer(_a							,					return_tensors='pt'							,					padding=_a )
       _SCREAMING_SNAKE_CASE							    =inputs['input_ids'].to(_a )

       _SCREAMING_SNAKE_CASE							    =model.generate(
           input_ids=_a							,					attention_mask=inputs['attention_mask'].to(_a )							,					)

       _SCREAMING_SNAKE_CASE							    =tokenizer(sentences[0]							,					return_tensors='pt' ).input_ids.to(_a )
       _SCREAMING_SNAKE_CASE							    =model.generate(input_ids=_a )

       _SCREAMING_SNAKE_CASE							    =inputs_non_padded.shape[-1] - inputs['attention_mask'][-1].long().sum().cpu().item()
       _SCREAMING_SNAKE_CASE							    =tokenizer(sentences[1]							,					return_tensors='pt' ).input_ids.to(_a )
       _SCREAMING_SNAKE_CASE							    =model.generate(input_ids=_a							,					max_length=model.config.max_length - num_paddings )

       _SCREAMING_SNAKE_CASE							    =tokenizer.batch_decode(_a							,					skip_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(output_non_padded[0]							,					skip_special_tokens=_a )
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(output_padded[0]							,					skip_special_tokens=_a )

       _SCREAMING_SNAKE_CASE							    =[
           'Hello, my dog is a little bit bigger than a little bit.',
           'Today, I have a good idea of how to use the information',
       ]
       self.assertListEqual(_a							,					_a )
       self.assertListEqual(_a							,					[non_padded_sentence, padded_sentence] )





  @slow
  def 						A  (		self      :	str )	->    Any:





       '''simple docstring'''






       for model_name in BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            _SCREAMING_SNAKE_CASE							    =BioGptModel.from_pretrained(_a )
            self.assertIsNotNone(_a )





  def 						A  (		self      :	List[str] )	->    Union[str, Any]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs_for_common()
       _SCREAMING_SNAKE_CASE							    =3
       _SCREAMING_SNAKE_CASE							    =input_dict['input_ids']
       _SCREAMING_SNAKE_CASE							    =input_ids.ne(1 ).to(_a )
       _SCREAMING_SNAKE_CASE							    =ids_tensor([self.model_tester.batch_size]							,					self.model_tester.type_sequence_label_size )
       _SCREAMING_SNAKE_CASE							    =BioGptForSequenceClassification(_a )
       model.to(_a )
       model.eval()
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					labels=_a )
       self.assertEqual(result.logits.shape							,					(self.model_tester.batch_size, self.model_tester.num_labels) )





  def 						A  (		self      :	Tuple )	->    List[str]:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE							    =self.model_tester.prepare_config_and_inputs_for_common()
       _SCREAMING_SNAKE_CASE							    =3
       _SCREAMING_SNAKE_CASE							    ='multi_label_classification'
       _SCREAMING_SNAKE_CASE							    =input_dict['input_ids']
       _SCREAMING_SNAKE_CASE							    =input_ids.ne(1 ).to(_a )
       _SCREAMING_SNAKE_CASE							    =ids_tensor(
           [self.model_tester.batch_size, config.num_labels]							,					self.model_tester.type_sequence_label_size ).to(torch.float )
       _SCREAMING_SNAKE_CASE							    =BioGptForSequenceClassification(_a )
       model.to(_a )
       model.eval()
       _SCREAMING_SNAKE_CASE							    =model(_a							,					attention_mask=_a							,					labels=_a )
       self.assertEqual(result.logits.shape							,					(self.model_tester.batch_size, self.model_tester.num_labels) )







@require_torch
class 							A__    (       unittest.TestCase  ):





  @slow
  def 						A  (		self      :	Any )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptForCausalLM.from_pretrained('microsoft/biogpt' )
       _SCREAMING_SNAKE_CASE							    =torch.tensor([[2, 4805, 9, 656, 21]] )
       _SCREAMING_SNAKE_CASE							    =model(_a )[0]

       _SCREAMING_SNAKE_CASE							    =4_2384

       _SCREAMING_SNAKE_CASE							    =torch.Size((1, 5, vocab_size) )
       self.assertEqual(output.shape							,					_a )

       _SCREAMING_SNAKE_CASE							    =torch.tensor(
           [[[-9.52_36, -9.89_18, 10.45_57], [-11.04_69, -9.64_23, 8.10_22], [-8.86_64, -7.88_26, 5.53_25]]] )

       self.assertTrue(torch.allclose(output[:, :3, :3]							,					_a							,					atol=1e-4 ) )





  @slow
  def 						A  (		self      :	Any )	->    Any:





       '''simple docstring'''






       _SCREAMING_SNAKE_CASE							    =BioGptTokenizer.from_pretrained('microsoft/biogpt' )
       _SCREAMING_SNAKE_CASE							    =BioGptForCausalLM.from_pretrained('microsoft/biogpt' )
       model.to(_a )

       torch.manual_seed(0 )
       _SCREAMING_SNAKE_CASE							    =tokenizer('COVID-19 is'							,					return_tensors='pt' ).to(_a )
       _SCREAMING_SNAKE_CASE							    =model.generate(
           **_a							,					min_length=100							,					max_length=1024							,					num_beams=5							,					early_stopping=_a							,					)
       _SCREAMING_SNAKE_CASE							    =tokenizer.decode(output_ids[0]							,					skip_special_tokens=_a )

       _SCREAMING_SNAKE_CASE							    =(
           'COVID-19 is a global pandemic caused by severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2), the'
           ' causative agent of coronavirus disease 2019 (COVID-19), which has spread to more than 200 countries and'
           ' territories, including the United States (US), Canada, Australia, New Zealand, the United Kingdom (UK),'
           ' and the United States of America (USA), as of March 11, 2020, with more than 800,000 confirmed cases and'
           ' more than 800,000 deaths.'
       )
       self.assertEqual(_a							,					_a )



'''simple docstring'''



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase :       Optional[int]			   =     get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCAmelCase :       Optional[int]			   =     2_5_6_0_4_7
UpperCAmelCase :       Union[str, Any]			   =     2_5_6_1_4_5







@require_sentencepiece
@require_tokenizers
class 						lowerCAmelCase__   (      a ,					unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      NllbTokenizer
 lowerCAmelCase__  					=      NllbTokenizerFast
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      True
 lowerCAmelCase__  					=      {}
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> int:


      """simple docstring"""




      super().setUp()

      # We have a SentencePiece fixture for testing
      __SCREAMING_SNAKE_CASE     =      NllbTokenizer(__SCREAMING_SNAKE_CASE					,							keep_accents=__SCREAMING_SNAKE_CASE			)
      tokenizer.save_pretrained(self.tmpdirname			)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      NllbTokenizer(__SCREAMING_SNAKE_CASE					,							keep_accents=__SCREAMING_SNAKE_CASE			)

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize("""This is a test"""			)
      self.assertListEqual(__SCREAMING_SNAKE_CASE					,							["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE			)					,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,							)

      __SCREAMING_SNAKE_CASE     =      tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """9""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """é""",
              """.""",
          ]					,							)
      __SCREAMING_SNAKE_CASE     =      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]					,							)

      __SCREAMING_SNAKE_CASE     =      tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE			)
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE					,							[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """<unk>""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """<unk>""",
              """.""",
          ]					,							)
 def       UpperCAmelCase__		(		self					:       Dict			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)

                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)
                __SCREAMING_SNAKE_CASE     =      tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f			)
                self.assertSequenceEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)

                # Save tokenizer rust, legacy_format=True
                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE					,							legacy_format=__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it save with the same files
                self.assertSequenceEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)

                # Save tokenizer rust, legacy_format=False
                __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.save_pretrained(__SCREAMING_SNAKE_CASE					,							legacy_format=__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.save_pretrained(__SCREAMING_SNAKE_CASE			)

                # Checks it saved the tokenizer.json file
                self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files			)			)

                # Checks everything loads correctly in the same way
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.from_pretrained(__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_p.from_pretrained(__SCREAMING_SNAKE_CASE			)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)			)

                shutil.rmtree(__SCREAMING_SNAKE_CASE			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       List[Any]			)  -> Any:


      """simple docstring"""




      if not self.test_seqaseq:
           return

      __SCREAMING_SNAKE_CASE     =      self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'			):
                # Longer text that will definitely require truncation.
                __SCREAMING_SNAKE_CASE     =      [
                    """ UN Chief Says There Is No Military Solution in Syria""",
                    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
                    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
                    """ will only worsen the violence and misery for millions of people.""",
                ]
                __SCREAMING_SNAKE_CASE     =      [
                    """Şeful ONU declară că nu există o soluţie militară în Siria""",
                    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
                    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
                    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
                ]
                try:
                     __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                         src_texts=__SCREAMING_SNAKE_CASE					,							tgt_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							max_target_length=10					,							return_tensors="""pt"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""ron_Latn"""					,							)
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]					,							3			)
                self.assertEqual(batch.labels.shape[1]					,							10			)
                # max_target_length will default to max_length if not specified
                __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                    __SCREAMING_SNAKE_CASE					,							tgt_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							return_tensors="""pt"""			)
                self.assertEqual(batch.input_ids.shape[1]					,							3			)
                self.assertEqual(batch.labels.shape[1]					,							3			)

                __SCREAMING_SNAKE_CASE     =      tokenizer.prepare_seqaseq_batch(
                    src_texts=__SCREAMING_SNAKE_CASE					,							max_length=3					,							max_target_length=10					,							return_tensors="""pt"""			)
                self.assertEqual(batch_encoder_only.input_ids.shape[1]					,							3			)
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]					,							3			)
                self.assertNotIn("""decoder_input_ids"""					,							__SCREAMING_SNAKE_CASE			)
 @unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece."""			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[int]:


      """simple docstring"""




      pass


 def       UpperCAmelCase__		(		self					:       List[str]			)  -> Dict:


      """simple docstring"""




      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                __SCREAMING_SNAKE_CASE     =      [AddedToken("""<special>"""					,							lstrip=__SCREAMING_SNAKE_CASE			)]

                __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(
                    __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)
                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode("""Hey this is a <special> token"""			)

                __SCREAMING_SNAKE_CASE     =      tokenizer_r.encode("""<special>"""					,							add_special_tokens=__SCREAMING_SNAKE_CASE			)[0]

                self.assertTrue(special_token_id in r_output			)

                if self.test_slow_tokenizer:
                     __SCREAMING_SNAKE_CASE     =      self.rust_tokenizer_class.from_pretrained(
                         __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE					,							)
                     __SCREAMING_SNAKE_CASE     =      self.tokenizer_class.from_pretrained(
                         __SCREAMING_SNAKE_CASE					,							additional_special_tokens=__SCREAMING_SNAKE_CASE					,							**__SCREAMING_SNAKE_CASE			)

                     __SCREAMING_SNAKE_CASE     =      tokenizer_p.encode("""Hey this is a <special> token"""			)

                     __SCREAMING_SNAKE_CASE     =      tokenizer_cr.encode("""Hey this is a <special> token"""			)

                     self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
                     self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
                     self.assertTrue(special_token_id in p_output			)
                     self.assertTrue(special_token_id in cr_output			)







@require_torch
@require_sentencepiece
@require_tokenizers
class 						lowerCAmelCase__   (      unittest.TestCase					):



 """simple docstring"""



 lowerCAmelCase__  					=      "facebook/nllb-200-distilled-600M"
 lowerCAmelCase__  					=      [
     " UN Chief Says There Is No Military Solution in Syria",
     " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
 ]
 lowerCAmelCase__  					=      [
     "Şeful ONU declară că nu există o soluţie militară în Siria",
     "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
     " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
     " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
 ]
 lowerCAmelCase__  					=      [
     256047,
     16297,
     134408,
     8165,
     248066,
     14734,
     950,
     1135,
     105721,
     3573,
     83,
     27352,
     108,
     49486,
     2,
 ]
 @classmethod
 def       UpperCAmelCase__		(		cls					:       List[Any]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      NllbTokenizer.from_pretrained(
          cls.checkpoint_name					,							src_lang="""eng_Latn"""					,							tgt_lang="""ron_Latn"""			)
      __SCREAMING_SNAKE_CASE     =      1
      return cls
 def       UpperCAmelCase__		(		self					:       Union[str, Any]			)  -> List[str]:


      """simple docstring"""




      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]					,							256_001			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]					,							256_002			)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]					,							256_057			)
 def       UpperCAmelCase__		(		self					:       Optional[int]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
      self.assertListEqual(self.expected_src_tokens					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Any			)  -> int:


      """simple docstring"""




      self.assertIn(__SCREAMING_SNAKE_CASE					,							self.tokenizer.all_special_ids			)
      # fmt: off
      __SCREAMING_SNAKE_CASE     =      [RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
      # fmt: on

      __SCREAMING_SNAKE_CASE     =      self.tokenizer.decode(__SCREAMING_SNAKE_CASE					,							skip_special_tokens=__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      self.tokenizer.decode(generated_ids[1:]					,							skip_special_tokens=__SCREAMING_SNAKE_CASE			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
      self.assertNotIn(self.tokenizer.eos_token					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      ["""this is gunna be a long sentence """ * 20]
      assert isinstance(src_text[0]					,							__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      10
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(__SCREAMING_SNAKE_CASE					,							max_length=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE			).input_ids[0]
      self.assertEqual(ids[-1]					,							2			)
      self.assertEqual(ids[0]					,							__SCREAMING_SNAKE_CASE			)
      self.assertEqual(len(__SCREAMING_SNAKE_CASE			)					,							__SCREAMING_SNAKE_CASE			)
 def       UpperCAmelCase__		(		self					:       int			)  -> List[Any]:


      """simple docstring"""




      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]			)					,							[256_203, 3]			)
 def       UpperCAmelCase__		(		self					:       str			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      tempfile.mkdtemp()
      __SCREAMING_SNAKE_CASE     =      self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      NllbTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE			)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,							__SCREAMING_SNAKE_CASE			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       str			)  -> List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          self.src_text					,							text_target=self.tgt_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=len(self.expected_src_tokens			)					,							return_tensors="""pt"""					,							)
      __SCREAMING_SNAKE_CASE     =      shift_tokens_right(
          batch["""labels"""]					,							self.tokenizer.pad_token_id					,							self.tokenizer.lang_code_to_id["""ron_Latn"""]			)

      self.assertIsInstance(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)

      self.assertEqual((2, 15)					,							batch.input_ids.shape			)
      self.assertEqual((2, 15)					,							batch.attention_mask.shape			)
      __SCREAMING_SNAKE_CASE     =      batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens					,							__SCREAMING_SNAKE_CASE			)
      self.assertEqual(__SCREAMING_SNAKE_CASE					,							batch.decoder_input_ids[0, 0]			)  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens					,							[EN_CODE]			)
      self.assertEqual(self.tokenizer.suffix_tokens					,							[self.tokenizer.eos_token_id]			)
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer(self.src_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=3					,							return_tensors="""pt"""			)
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          text_target=self.tgt_text					,							padding=__SCREAMING_SNAKE_CASE					,							truncation=__SCREAMING_SNAKE_CASE					,							max_length=10					,							return_tensors="""pt"""			)
      __SCREAMING_SNAKE_CASE     =      targets["""input_ids"""]
      __SCREAMING_SNAKE_CASE     =      shift_tokens_right(
          __SCREAMING_SNAKE_CASE					,							self.tokenizer.pad_token_id					,							decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					,							)

      self.assertEqual(batch.input_ids.shape[1]					,							3			)
      self.assertEqual(batch.decoder_input_ids.shape[1]					,							10			)
 @require_torch
 def       UpperCAmelCase__		(		self					:       int			)  -> str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      self.tokenizer._build_translation_inputs(
          """A test"""					,							return_tensors="""pt"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)

      self.assertEqual(
          nested_simplify(__SCREAMING_SNAKE_CASE			)					,							{
              # A, test, EOS, en_XX
              """input_ids""": [[256_047, 70, 7_356, 2]],
              """attention_mask""": [[1, 1, 1, 1]],
              # ar_AR
              """forced_bos_token_id""": 256_057,
          }					,							)


 @require_torch
 def       UpperCAmelCase__		(		self					:       Optional[Any]			)  -> Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      True
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          """UN Chief says there is no military solution in Syria"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)
      self.assertEqual(
          inputs.input_ids					,							[16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047]			)

      __SCREAMING_SNAKE_CASE     =      False
      __SCREAMING_SNAKE_CASE     =      self.tokenizer(
          """UN Chief says there is no military solution in Syria"""					,							src_lang="""eng_Latn"""					,							tgt_lang="""fra_Latn"""			)
      self.assertEqual(
          inputs.input_ids					,							[256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2]			)






