
import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_PRETRAINING_MAPPING,
        MobileBertForMaskedLM,
        MobileBertForMultipleChoice,
        MobileBertForNextSentencePrediction,
        MobileBertForPreTraining,
        MobileBertForQuestionAnswering,
        MobileBertForSequenceClassification,
        MobileBertForTokenClassification,
        MobileBertModel,
    )




class 	__A:

  """simple docstring"""







  def __init__(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_=13       ,			SCREAMING_SNAKE_CASE_=7       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=99       ,			SCREAMING_SNAKE_CASE_=64       ,			SCREAMING_SNAKE_CASE_=32       ,			SCREAMING_SNAKE_CASE_=5       ,			SCREAMING_SNAKE_CASE_=4       ,			SCREAMING_SNAKE_CASE_=37       ,			SCREAMING_SNAKE_CASE_="gelu"       ,			SCREAMING_SNAKE_CASE_=0.1       ,			SCREAMING_SNAKE_CASE_=0.1       ,			SCREAMING_SNAKE_CASE_=5_12       ,			SCREAMING_SNAKE_CASE_=16       ,			SCREAMING_SNAKE_CASE_=2       ,			SCREAMING_SNAKE_CASE_=0.02       ,			SCREAMING_SNAKE_CASE_=3       ,			SCREAMING_SNAKE_CASE_=4       ,			SCREAMING_SNAKE_CASE_=None       ,			):
       UpperCamelCase__				=				parent
       UpperCamelCase__				=				batch_size
       UpperCamelCase__				=				seq_length
       UpperCamelCase__				=				is_training
       UpperCamelCase__				=				use_input_mask
       UpperCamelCase__				=				use_token_type_ids
       UpperCamelCase__				=				use_labels
       UpperCamelCase__				=				vocab_size
       UpperCamelCase__				=				hidden_size
       UpperCamelCase__				=				embedding_size
       UpperCamelCase__				=				num_hidden_layers
       UpperCamelCase__				=				num_attention_heads
       UpperCamelCase__				=				intermediate_size
       UpperCamelCase__				=				hidden_act
       UpperCamelCase__				=				hidden_dropout_prob
       UpperCamelCase__				=				attention_probs_dropout_prob
       UpperCamelCase__				=				max_position_embeddings
       UpperCamelCase__				=				type_vocab_size
       UpperCamelCase__				=				type_sequence_label_size
       UpperCamelCase__				=				initializer_range
       UpperCamelCase__				=				num_labels
       UpperCamelCase__				=				num_choices
       UpperCamelCase__				=				scope







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size						)

       UpperCamelCase__				=				None
       if self.use_input_mask:
            UpperCamelCase__				=				random_attention_mask([self.batch_size, self.seq_length]						)

       UpperCamelCase__				=				None
       if self.use_token_type_ids:
            UpperCamelCase__				=				ids_tensor([self.batch_size, self.seq_length]       ,			self.type_vocab_size						)

       UpperCamelCase__				=				None
       UpperCamelCase__				=				None
       UpperCamelCase__				=				None
       if self.use_labels:
            UpperCamelCase__				=				ids_tensor([self.batch_size]       ,			self.type_sequence_label_size						)
            UpperCamelCase__				=				ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels						)
            UpperCamelCase__				=				ids_tensor([self.batch_size]       ,			self.num_choices						)

       UpperCamelCase__				=				self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def        UpperCAmelCase_					(self						):
       return MobileBertConfig(
           vocab_size=self.vocab_size       ,			hidden_size=self.hidden_size       ,			num_hidden_layers=self.num_hidden_layers       ,			num_attention_heads=self.num_attention_heads       ,			intermediate_size=self.intermediate_size       ,			embedding_size=self.embedding_size       ,			hidden_act=self.hidden_act       ,			hidden_dropout_prob=self.hidden_dropout_prob       ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,			max_position_embeddings=self.max_position_embeddings       ,			type_vocab_size=self.type_vocab_size       ,			is_decoder=SCREAMING_SNAKE_CASE_       ,			initializer_range=self.initializer_range       ,			)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				MobileBertModel(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_						)

       self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape       ,			(self.batch_size, self.hidden_size)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				MobileBertForMaskedLM(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_						)
       self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				MobileBertForNextSentencePrediction(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(
           SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_       ,			)
       self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, 2)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				MobileBertForPreTraining(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(
           SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_       ,			next_sentence_label=SCREAMING_SNAKE_CASE_       ,			)
       self.parent.assertEqual(result.prediction_logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)						)
       self.parent.assertEqual(result.seq_relationship_logits.shape       ,			(self.batch_size, 2)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				MobileBertForQuestionAnswering(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(
           SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			start_positions=SCREAMING_SNAKE_CASE_       ,			end_positions=SCREAMING_SNAKE_CASE_       ,			)
       self.parent.assertEqual(result.start_logits.shape       ,			(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape       ,			(self.batch_size, self.seq_length)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				self.num_labels
       UpperCamelCase__				=				MobileBertForSequenceClassification(SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_						)
       self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.num_labels)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				self.num_labels
       UpperCamelCase__				=				MobileBertForTokenClassification(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_						)
       self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.num_labels)						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				self.num_choices
       UpperCamelCase__				=				MobileBertForMultipleChoice(config=SCREAMING_SNAKE_CASE_						)
       model.to(SCREAMING_SNAKE_CASE_						)
       model.eval()
       UpperCamelCase__				=				input_ids.unsqueeze(1						).expand(-1       ,			self.num_choices       ,			-1						).contiguous()
       UpperCamelCase__				=				token_type_ids.unsqueeze(1						).expand(-1       ,			self.num_choices       ,			-1						).contiguous()
       UpperCamelCase__				=				input_mask.unsqueeze(1						).expand(-1       ,			self.num_choices       ,			-1						).contiguous()
       UpperCamelCase__				=				model(
           SCREAMING_SNAKE_CASE_       ,			attention_mask=SCREAMING_SNAKE_CASE_       ,			token_type_ids=SCREAMING_SNAKE_CASE_       ,			labels=SCREAMING_SNAKE_CASE_       ,			)
       self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.num_choices)						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.prepare_config_and_inputs()
       (
           UpperCamelCase__   
       )				=				config_and_inputs
       UpperCamelCase__				=				{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
       return config, inputs_dict






@require_torch
class 	__A(				__lowerCamelCase				,      __lowerCamelCase				,      unittest.TestCase ):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__			    =			(
      (
          MobileBertModel,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  SCREAMING_SNAKE_CASE__			    =			(
      {
          """feature-extraction""": MobileBertModel,
          """fill-mask""": MobileBertForMaskedLM,
          """question-answering""": MobileBertForQuestionAnswering,
          """text-classification""": MobileBertForSequenceClassification,
          """token-classification""": MobileBertForTokenClassification,
          """zero-shot""": MobileBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  SCREAMING_SNAKE_CASE__			    =			True







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_=False						):
       UpperCamelCase__				=				super()._prepare_for_class(SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_       ,			return_labels=SCREAMING_SNAKE_CASE_						)

       if return_labels:
            if model_class in get_values(SCREAMING_SNAKE_CASE_						):
                 UpperCamelCase__				=				torch.zeros(
                     (self.model_tester.batch_size, self.model_tester.seq_length)       ,			dtype=torch.long       ,			device=SCREAMING_SNAKE_CASE_						)
                 UpperCamelCase__				=				torch.zeros(
                     self.model_tester.batch_size       ,			dtype=torch.long       ,			device=SCREAMING_SNAKE_CASE_						)
       return inputs_dict







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				MobileBertModelTester(self						)
       UpperCamelCase__				=				ConfigTester(self       ,			config_class=SCREAMING_SNAKE_CASE_       ,			hidden_size=37						)







  def        UpperCAmelCase_					(self						):
       self.config_tester.run_common_tests()







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_model(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_masked_lm(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_multiple_choice(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_pretraining(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_question_answering(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_sequence_classification(*SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_mobilebert_for_token_classification(*SCREAMING_SNAKE_CASE_						)






def 						__magic_name__ (    __a			:       Union[str, Any]						):



     '''simple docstring'''




     return torch.tensor(
         SCREAMING_SNAKE_CASE__       ,		dtype=torch.long       ,		device=SCREAMING_SNAKE_CASE__       ,		)


lowerCamelCase_							=     1E-3




@require_torch
@require_sentencepiece
@require_tokenizers
class 	__A(				unittest.TestCase ):

  """simple docstring"""







  @slow
  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				MobileBertModel.from_pretrained("""google/mobilebert-uncased"""						).to(SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]						)
       with torch.no_grad():
            UpperCamelCase__				=				model(SCREAMING_SNAKE_CASE_						)[0]
       UpperCamelCase__				=				torch.Size((1, 9, 5_12)						)
       self.assertEqual(output.shape       ,			SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				torch.tensor(
           [
               [
                   [-2.4736526E07, 8.2691656E04, 1.6521838E05],
                   [-5.7541704E-01, 3.9056022E00, 4.4011507E00],
                   [2.6047359E00, 1.5677652E00, -1.7324188E-01],
               ]
           ]       ,			device=SCREAMING_SNAKE_CASE_       ,			)

       # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
       # ~1 difference, it's therefore not a good idea to measure using addition.
       # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
       # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
       UpperCamelCase__				=				torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE						)
       UpperCamelCase__				=				torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE						)

       self.assertTrue(lower_bound and upper_bound						)












import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     GPTSanJapaneseTokenizer
       lowerCAmelCase__      		=     False
       lowerCAmelCase__      		=     {"""do_clean_text""": False, """add_prefix_space""": False}




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         super().setUp()

         # fmt: off
         __snake_case:						str   =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         __snake_case:						List[Any]   =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         __snake_case:						Optional[int]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						Any   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						int   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""emoji_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
         with open(self.emoji_file							,				"""w"""		) as emoji_writer:
           emoji_writer.write(json.dumps(A		)		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Dict		):
         __snake_case:						Tuple   =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
         __snake_case:						str   =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Optional[int]		):
         __snake_case  ,   __snake_case:						Optional[int]   =						self.get_input_output_texts(A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_special_tokens=A		)
         __snake_case:						int   =						tokenizer.decode(A							,				clean_up_tokenization_spaces=A		)
         return text, ids




       def 							UpperCAmelCase__ (  self :    int		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Dict		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Dict   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						List[str]   =						"""こんにちは、世界。　こんばんは、㔺界。"""
         __snake_case:						Optional[int]   =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         __snake_case:						Any   =						tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						str   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						Optional[int]   =						tokens + [tokenizer.unk_token]
         __snake_case:						List[str]   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    str		):
         __snake_case:						Union[str, Any]   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						Optional[Any]   =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         __snake_case:						Optional[Any]   =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
         __snake_case:						Union[str, Any]   =						tokenizer.encode(A		)
         __snake_case:						List[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Any		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Any   =						"""こんにちは、世界。"""
         __snake_case:						Tuple   =						"""こんばんは、㔺界。😀"""
         __snake_case:						Optional[Any]   =						"""こんにちは、世界。こんばんは、世界。😀"""
         __snake_case:						int   =						tokenizer.encode(prefix_text + input_text		)
         __snake_case:						Union[str, Any]   =						tokenizer.encode(""""""							,				prefix_text=prefix_text + input_text		)
         __snake_case:						Tuple   =						tokenizer.encode(A							,				prefix_text=A		)
         __snake_case:						Union[str, Any]   =						tokenizer.decode(A		)
         __snake_case:						Dict   =						tokenizer.decode(A		)
         __snake_case:						Optional[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						int   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Optional[int]   =						"""こんにちは、世界。"""
         __snake_case:						Any   =						"""こんばんは、㔺界。😀"""

         __snake_case:						Optional[int]   =						len(tokenizer.encode(A		)		) - 2
         __snake_case:						str   =						len(tokenizer.encode(A		)		) - 2

         __snake_case:						Dict   =						[1] + [0] * (len_prefix + len_text + 1)
         __snake_case:						str   =						[1] * (len_prefix + len_text + 1) + [0]
         __snake_case:						List[Any]   =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __snake_case:						int   =						tokenizer(prefix_text + input_text		).token_type_ids
         __snake_case:						Optional[int]   =						tokenizer(""""""							,				prefix_text=prefix_text + input_text		).token_type_ids
         __snake_case:						Tuple   =						tokenizer(A							,				prefix_text=A		).token_type_ids
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Dict		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						int   =						tokenizer.encode("""あンいワ"""		)
         __snake_case:						Optional[int]   =						tokenizer.encode(""""""							,				prefix_text="""あンいワ"""		)
         __snake_case:						List[Any]   =						tokenizer.encode("""いワ"""							,				prefix_text="""あン"""		)

         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertNotEqual(A							,				A		)
         self.assertNotEqual(A							,				A		)
         self.assertEqual(x_token_a[1]							,				x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]							,				x_token_a[3]		)  # SEG token




       @slow
       def 							UpperCAmelCase__ (  self :    int		):
         __snake_case:						List[str]   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						Union[str, Any]   =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         __snake_case:						Optional[int]   =						tokenizer(A							,				padding=A		)
         __snake_case:						int   =						tokenizer.batch_encode_plus(A							,				padding=A		)

         # fmt: off
         __snake_case:						List[str]   =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
         __snake_case:						int   =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __snake_case:						Dict   =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids							,				A		)
         self.assertListEqual(x_token.token_type_ids							,				A		)
         self.assertListEqual(x_token.attention_mask							,				A		)
         self.assertListEqual(x_token_a.input_ids							,				A		)
         self.assertListEqual(x_token_a.token_type_ids							,				A		)
         self.assertListEqual(x_token_a.attention_mask							,				A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






       def 							UpperCAmelCase__ (  self :    List[str]		):
         # tokenizer has no padding token
         pass






