


import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotSmallConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
	import os

	# The slow tests are often failing with OOM error on GPU
	# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
	# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
	A_     :       Tuple   							="""platform"""

	import jax
	import jax.numpy as jnp

	from transformers.models.blenderbot_small.modeling_flax_blenderbot_small import (
	    FlaxBlenderbotSmallForConditionalGeneration,
	    FlaxBlenderbotSmallModel,
	    shift_tokens_right,
	)




def 	lowerCamelCase_							(       UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__=None							,     UpperCAmelCase__=None							,     UpperCAmelCase__=None							,     UpperCAmelCase__=None							,     UpperCAmelCase__=None							,     UpperCAmelCase__=None							,     ):



						"""simple docstring"""




						if attention_mask is None:
												a_		      =		np.where(input_ids != config.pad_token_id							,     1							,     0   )
						if decoder_attention_mask is None:
												a_		      =		np.where(decoder_input_ids != config.pad_token_id							,     1							,     0   )
						if head_mask is None:
												a_		      =		np.ones((config.encoder_layers, config.encoder_attention_heads)   )
						if decoder_head_mask is None:
												a_		      =		np.ones((config.decoder_layers, config.decoder_attention_heads)   )
						if cross_attn_head_mask is None:
												a_		      =		np.ones((config.decoder_layers, config.decoder_attention_heads)   )
						return {
						    "input_ids": input_ids,
						    "decoder_input_ids": decoder_input_ids,
						    "attention_mask": attention_mask,
						    "decoder_attention_mask": attention_mask,
						}





class 							lowercase_					:






							"""simple docstring"""







							def __init__( self      ,     _UpperCAmelCase      ,     _UpperCAmelCase=13      ,     _UpperCAmelCase=7      ,     _UpperCAmelCase=True      ,     _UpperCAmelCase=False      ,     _UpperCAmelCase=99      ,     _UpperCAmelCase=16      ,     _UpperCAmelCase=2      ,     _UpperCAmelCase=4      ,     _UpperCAmelCase=4      ,     _UpperCAmelCase="gelu"      ,     _UpperCAmelCase=0.1      ,     _UpperCAmelCase=0.1      ,     _UpperCAmelCase=32      ,     _UpperCAmelCase=2      ,     _UpperCAmelCase=1      ,     _UpperCAmelCase=0      ,     _UpperCAmelCase=0.0_2      ,     ):







													"""simple docstring"""


													a_		      =		parent
													a_		      =		batch_size
													a_		      =		seq_length
													a_		      =		is_training
													a_		      =		use_labels
													a_		      =		vocab_size
													a_		      =		hidden_size
													a_		      =		num_hidden_layers
													a_		      =		num_attention_heads
													a_		      =		intermediate_size
													a_		      =		hidden_act
													a_		      =		hidden_dropout_prob
													a_		      =		attention_probs_dropout_prob
													a_		      =		max_position_embeddings
													a_		      =		eos_token_id
													a_		      =		pad_token_id
													a_		      =		bos_token_id
													a_		      =		initializer_range


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		np.clip(ids_tensor([self.batch_size, self.seq_length - 1]      ,     self.vocab_size		)      ,     3      ,     self.vocab_size		)
													a_		      =		np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)      ,     dtype=np.intaa		))      ,     -1		)

													a_		      =		shift_tokens_right(lowerCamelCase__      ,     1      ,     2		)

													a_		      =		BlenderbotSmallConfig(
													    vocab_size=self.vocab_size      ,     d_model=self.hidden_size      ,     encoder_layers=self.num_hidden_layers      ,     decoder_layers=self.num_hidden_layers      ,     encoder_attention_heads=self.num_attention_heads      ,     decoder_attention_heads=self.num_attention_heads      ,     encoder_ffn_dim=self.intermediate_size      ,     decoder_ffn_dim=self.intermediate_size      ,     dropout=self.hidden_dropout_prob      ,     attention_dropout=self.attention_probs_dropout_prob      ,     max_position_embeddings=self.max_position_embeddings      ,     eos_token_id=self.eos_token_id      ,     bos_token_id=self.bos_token_id      ,     pad_token_id=self.pad_token_id      ,     initializer_range=self.initializer_range      ,     use_cache=lowerCamelCase__      ,     )
													a_		      =		prepare_blenderbot_inputs_dict(lowerCamelCase__      ,     lowerCamelCase__      ,     lowerCamelCase__		)
													return config, inputs_dict


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self.prepare_config_and_inputs()
													return config, inputs_dict


							def     lowercase__  ( self      ,     _UpperCAmelCase      ,     _UpperCAmelCase      ,     _UpperCAmelCase		):







													"""simple docstring"""


													a_		      =		20
													a_		      =		model_class_name(lowerCamelCase__		)

													a_		      =		model.encode(inputs_dict["""input_ids"""]		)

													a_		      =		(
													    inputs_dict["decoder_input_ids"],
													    inputs_dict["decoder_attention_mask"],
													)

													a_		      =		model.init_cache(decoder_input_ids.shape[0]      ,     lowerCamelCase__      ,     lowerCamelCase__		)
													a_		      =		jnp.ones((decoder_input_ids.shape[0], max_decoder_length)      ,     dtype="""i4"""		)

													a_		      =		jnp.broadcast_to(
													    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]      ,     (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)      ,     )
													a_		      =		model.decode(
													    decoder_input_ids[:, :-1]      ,     lowerCamelCase__      ,     decoder_attention_mask=lowerCamelCase__      ,     past_key_values=lowerCamelCase__      ,     decoder_position_ids=lowerCamelCase__      ,     )

													a_		      =		jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]      ,     dtype="""i4"""		)
													a_		      =		model.decode(
													    decoder_input_ids[:, -1:]      ,     lowerCamelCase__      ,     decoder_attention_mask=lowerCamelCase__      ,     past_key_values=outputs_cache.past_key_values      ,     decoder_position_ids=lowerCamelCase__      ,     )

													a_		      =		model.decode(lowerCamelCase__      ,     lowerCamelCase__		)

													a_		      =		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
													self.parent.assertTrue(diff < 1e-3      ,     msg=f"Max diff is {diff}"		)






							def     lowercase__  ( self      ,     _UpperCAmelCase      ,     _UpperCAmelCase      ,     _UpperCAmelCase		):







													"""simple docstring"""


													a_		      =		20
													a_		      =		model_class_name(lowerCamelCase__		)

													a_		      =		model.encode(inputs_dict["""input_ids"""]		)

													a_		      =		(
													    inputs_dict["decoder_input_ids"],
													    inputs_dict["decoder_attention_mask"],
													)

													a_		      =		jnp.concatenate(
													    [
													        decoder_attention_mask,
													        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])		),
													    ]      ,     axis=-1      ,     )

													a_		      =		model.init_cache(decoder_input_ids.shape[0]      ,     lowerCamelCase__      ,     lowerCamelCase__		)
													a_		      =		jnp.broadcast_to(
													    jnp.arange(decoder_input_ids.shape[-1] - 1		)[None, :]      ,     (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)      ,     )

													a_		      =		model.decode(
													    decoder_input_ids[:, :-1]      ,     lowerCamelCase__      ,     decoder_attention_mask=lowerCamelCase__      ,     past_key_values=lowerCamelCase__      ,     decoder_position_ids=lowerCamelCase__      ,     )
													a_		      =		jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]      ,     dtype="""i4"""		)
													a_		      =		model.decode(
													    decoder_input_ids[:, -1:]      ,     lowerCamelCase__      ,     past_key_values=outputs_cache.past_key_values      ,     decoder_attention_mask=lowerCamelCase__      ,     decoder_position_ids=lowerCamelCase__      ,     )

													a_		      =		model.decode(lowerCamelCase__      ,     lowerCamelCase__      ,     decoder_attention_mask=lowerCamelCase__		)

													a_		      =		np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
													self.parent.assertTrue(diff < 1e-3      ,     msg=f"Max diff is {diff}"		)





@require_flax
class 							lowercase_					(			unittest.TestCase):






							"""simple docstring"""







							snake_case_										=		99


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		np.array(
													    [
													        [71, 82, 18, 33, 46, 91, 2],
													        [68, 34, 26, 58, 30, 82, 2],
													        [5, 97, 17, 39, 94, 40, 2],
													        [76, 83, 94, 25, 70, 78, 2],
													        [87, 59, 41, 35, 48, 66, 2],
													        [55, 13, 16, 58, 5, 2, 1],  # note padding
													        [64, 27, 31, 51, 12, 75, 2],
													        [52, 64, 86, 17, 83, 39, 2],
													        [48, 61, 9, 24, 71, 82, 2],
													        [26, 1, 60, 48, 22, 13, 2],
													        [21, 5, 62, 28, 14, 76, 2],
													        [45, 98, 37, 86, 59, 48, 2],
													        [70, 70, 50, 9, 28, 0, 2],
													    ]      ,     dtype=np.intaa      ,     )

													a_		      =		input_ids.shape[0]
													a_		      =		BlenderbotSmallConfig(
													    vocab_size=self.vocab_size      ,     d_model=24      ,     encoder_layers=2      ,     decoder_layers=2      ,     encoder_attention_heads=2      ,     decoder_attention_heads=2      ,     encoder_ffn_dim=32      ,     decoder_ffn_dim=32      ,     max_position_embeddings=48      ,     eos_token_id=2      ,     pad_token_id=1      ,     bos_token_id=0      ,     )
													return config, input_ids, batch_size


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self._get_config_and_data()
													a_		      =		FlaxBlenderbotSmallForConditionalGeneration(lowerCamelCase__		)
													a_		      =		lm_model(input_ids=lowerCamelCase__		)
													a_		      =		(batch_size, input_ids.shape[1], config.vocab_size)
													self.assertEqual(outputs["""logits"""].shape      ,     lowerCamelCase__		)


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		BlenderbotSmallConfig(
													    vocab_size=self.vocab_size      ,     d_model=14      ,     encoder_layers=2      ,     decoder_layers=2      ,     encoder_attention_heads=2      ,     decoder_attention_heads=2      ,     encoder_ffn_dim=8      ,     decoder_ffn_dim=8      ,     max_position_embeddings=48      ,     )
													a_		      =		FlaxBlenderbotSmallForConditionalGeneration(lowerCamelCase__		)
													a_		      =		np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]]      ,     dtype=np.intaa		)
													a_		      =		np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]]      ,     dtype=np.intaa		)
													a_		      =		lm_model(input_ids=lowerCamelCase__      ,     decoder_input_ids=lowerCamelCase__		)
													a_		      =		(*summary.shape, config.vocab_size)
													self.assertEqual(outputs["""logits"""].shape      ,     lowerCamelCase__		)






							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]]      ,     dtype=np.intaa		)
													a_		      =		shift_tokens_right(lowerCamelCase__      ,     1      ,     2		)
													a_		      =		np.equal(lowerCamelCase__      ,     1		).astype(np.floataa		).sum()
													a_		      =		np.equal(lowerCamelCase__      ,     1		).astype(np.floataa		).sum()
													self.assertEqual(shifted.shape      ,     input_ids.shape		)
													self.assertEqual(lowerCamelCase__      ,     n_pad_before - 1		)
													self.assertTrue(np.equal(shifted[:, 0]      ,     2		).all()		)





@require_flax
class 							lowercase_					(			UpperCAmelCase__      ,unittest.TestCase      ,UpperCAmelCase__):






							"""simple docstring"""







							snake_case_										=		True
							snake_case_										=		(
							    (
							        FlaxBlenderbotSmallModel,
							        FlaxBlenderbotSmallForConditionalGeneration,
							    )
							    if is_flax_available()
							    else ()
							)
							snake_case_										=		(FlaxBlenderbotSmallForConditionalGeneration,) if is_flax_available() else ()


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		FlaxBlenderbotSmallModelTester(self		)


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self.model_tester.prepare_config_and_inputs()
													for model_class in self.all_model_classes:
																			self.model_tester.check_use_cache_forward(lowerCamelCase__      ,     lowerCamelCase__      ,     lowerCamelCase__		)


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self.model_tester.prepare_config_and_inputs()
													for model_class in self.all_model_classes:
																			self.model_tester.check_use_cache_forward_with_attn_mask(lowerCamelCase__      ,     lowerCamelCase__      ,     lowerCamelCase__		)


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			with self.subTest(model_class.__name__		):
																									a_		      =		self._prepare_for_class(lowerCamelCase__      ,     lowerCamelCase__		)
																									a_		      =		model_class(lowerCamelCase__		)

																									@jax.jit
																									def encode_jitted(_UpperCAmelCase      ,     _UpperCAmelCase=None      ,     **_UpperCAmelCase		):
																															return model.encode(input_ids=lowerCamelCase__      ,     attention_mask=lowerCamelCase__		)

																									with self.subTest("""JIT Enabled"""		):
																															a_		      =		encode_jitted(**lowerCamelCase__		).to_tuple()

																									with self.subTest("""JIT Disabled"""		):
																															with jax.disable_jit():
																																					a_		      =		encode_jitted(**lowerCamelCase__		).to_tuple()

																									self.assertEqual(len(lowerCamelCase__		)      ,     len(lowerCamelCase__		)		)
																									for jitted_output, output in zip(lowerCamelCase__      ,     lowerCamelCase__		):
																															self.assertEqual(jitted_output.shape      ,     output.shape		)


							def     lowercase__  ( self		):







													"""simple docstring"""


													a_		      =		self.model_tester.prepare_config_and_inputs_for_common()

													for model_class in self.all_model_classes:
																			with self.subTest(model_class.__name__		):
																									a_		      =		model_class(lowerCamelCase__		)
																									a_		      =		model.encode(inputs_dict["""input_ids"""]      ,     inputs_dict["""attention_mask"""]		)

																									a_		      =		{
																									    "decoder_input_ids": inputs_dict["decoder_input_ids"],
																									    "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
																									    "encoder_outputs": encoder_outputs,
																									}

																									@jax.jit
																									def decode_jitted(_UpperCAmelCase      ,     _UpperCAmelCase      ,     _UpperCAmelCase		):
																															return model.decode(
																															    decoder_input_ids=lowerCamelCase__      ,     decoder_attention_mask=lowerCamelCase__      ,     encoder_outputs=lowerCamelCase__      ,     )

																									with self.subTest("""JIT Enabled"""		):
																															a_		      =		decode_jitted(**lowerCamelCase__		).to_tuple()

																									with self.subTest("""JIT Disabled"""		):
																															with jax.disable_jit():
																																					a_		      =		decode_jitted(**lowerCamelCase__		).to_tuple()

																									self.assertEqual(len(lowerCamelCase__		)      ,     len(lowerCamelCase__		)		)
																									for jitted_output, output in zip(lowerCamelCase__      ,     lowerCamelCase__		):
																															self.assertEqual(jitted_output.shape      ,     output.shape		)






							@slow
							def     lowercase__  ( self		):







													"""simple docstring"""


													for model_class_name in self.all_model_classes:
																			a_		      =		model_class_name.from_pretrained("""facebook/blenderbot_small-90M"""		)
																			# FlaxBlenderbotForSequenceClassification expects eos token in input_ids
																			a_		      =		np.ones((1, 1)		) * model.config.eos_token_id
																			a_		      =		model(lowerCamelCase__		)
																			self.assertIsNotNone(lowerCamelCase__		)
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Union[str, Any]          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Dict          =    False
    SCREAMING_SNAKE_CASE:				List[Any]          =    True




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							Optional[int]       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	"こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_  ,	lowerCAmelCase_:							str       =	self.get_input_output_texts(lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode(lowerCamelCase__       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							List[str]       =	tokenizer.decode(lowerCamelCase__       ,						clean_up_tokenization_spaces=lowerCamelCase__ )
       return text, ids




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	self.tokenizer_class(self.vocab_file )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。" )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="mecab" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[Any]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Dict       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Union[str, Any]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	MecabTokenizer(mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							List[Any]       =	MecabTokenizer(mecab_dic="unidic_lite" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							str       =	MecabTokenizer(mecab_dic="unidic" )
       except ModuleNotFoundError:
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							List[Any]       =	MecabTokenizer(do_lower_case=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    def      _a    (    self ):
       try:
          lowerCAmelCase_:							Any       =	MecabTokenizer(
              do_lower_case=lowerCamelCase__       ,						normalize_text=lowerCamelCase__       ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic" )
       except RuntimeError:
          # if dict doesn't exist in the system, previous code raises this error.
          return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	MecabTokenizer(normalize_text=lowerCamelCase__       ,						mecab_dic="ipadic" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="sudachi" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							str       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							List[str]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[int]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							List[str]       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Optional[int]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="A" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国", "人", "参政", "権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="B" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人", "参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	SudachiTokenizer(sudachi_dict_type="core"       ,						sudachi_split_mode="C" )

       self.assertListEqual(tokenizer.tokenize("外国人参政権" )       ,						["外国人参政権"] )




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	SudachiTokenizer(do_lower_case=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	SudachiTokenizer(normalize_text=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]       ,						)




    @require_sudachi
    def      _a    (    self ):
       lowerCAmelCase_:							Tuple       =	SudachiTokenizer(trim_whitespace=lowerCamelCase__       ,						sudachi_dict_type="core" )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							int       =	self.tokenizer_class(self.vocab_file       ,						word_tokenizer_type="jumanpp" )
       self.assertIsNotNone(lowerCamelCase__ )

       lowerCAmelCase_:							Any       =	"こんにちは、世界。\nこんばんは、世界。"
       lowerCAmelCase_:							Optional[int]       =	tokenizer.tokenize(lowerCamelCase__ )
       self.assertListEqual(lowerCamelCase__       ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

       lowerCAmelCase_:							Optional[Any]       =	os.path.join(self.tmpdirname       ,						"tokenizer.bin" )
       with open(lowerCamelCase__       ,						"wb" ) as handle:
          pickle.dump(lowerCamelCase__       ,						lowerCamelCase__ )

       with open(lowerCamelCase__       ,						"rb" ) as handle:
          lowerCAmelCase_:							Any       =	pickle.load(lowerCamelCase__ )

       lowerCAmelCase_:							Union[str, Any]       =	tokenizer_new.tokenize(lowerCamelCase__ )

       self.assertListEqual(lowerCamelCase__       ,						lowerCamelCase__ )




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(do_lower_case=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	JumanppTokenizer(normalize_text=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	JumanppTokenizer(trim_whitespace=lowerCamelCase__ )

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  " )       ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]       ,						)




    @require_jumanpp
    def      _a    (    self ):
       lowerCAmelCase_:							Any       =	JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。" )       ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]       ,						)




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[int]       =	["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       lowerCAmelCase_:							Tuple       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							List[Any]       =	i
       lowerCAmelCase_:							List[str]       =	WordpieceTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こんにちは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは" )       ,						["こん", "##ばんは"] )

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは" )       ,						["こん", "##ばんは", "[UNK]", "こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							List[str]       =	BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp" )
       lowerCAmelCase_:							Optional[Any]       =	tokenizer.subword_tokenizer

       lowerCAmelCase_:							List[str]       =	subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。" )
       self.assertListEqual(lowerCamelCase__       ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"] )

       lowerCAmelCase_:							Optional[int]       =	subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは" )
       self.assertListEqual(lowerCamelCase__       ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"] )




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese" )

       lowerCAmelCase_:							int       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Optional[int]       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							List[Any]       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     UpperCAmelCase__ ,  unittest.TestCase			):






    '''simple docstring'''
    SCREAMING_SNAKE_CASE:				Tuple          =    BertJapaneseTokenizer
    SCREAMING_SNAKE_CASE:				Optional[Any]          =    False




    def      _a    (    self ):
       super().setUp()

       lowerCAmelCase_:							Any       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							int       =	os.path.join(self.tmpdirname       ,						VOCAB_FILES_NAMES["vocab_file"] )
       with open(self.vocab_file       ,						"w"       ,						encoding="utf-8" ) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens] ) )




    def      _a    (    self       ,						**lowerCamelCase__ ):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,						subword_tokenizer_type="character"       ,						**lowerCamelCase__ )




    def      _a    (    self       ,						lowerCamelCase__ ):
       lowerCAmelCase_:							str       =	"こんにちは、世界。 \nこんばんは、世界。"
       lowerCAmelCase_:							Tuple       =	"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       pass  # TODO add if relevant




    def      _a    (    self ):
       lowerCAmelCase_:							str       =	self.tokenizer_class(self.vocab_file       ,						subword_tokenizer_type="character" )

       lowerCAmelCase_:							Any       =	tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。" )
       self.assertListEqual(
           lowerCamelCase__       ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"] )
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowerCamelCase__ )       ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       lowerCAmelCase_:							List[str]       =	{}
       for i, token in enumerate(lowerCamelCase__ ):
          lowerCAmelCase_:							Optional[Any]       =	i
       lowerCAmelCase_:							List[Any]       =	CharacterTokenizer(vocab=lowerCamelCase__       ,						unk_token="[UNK]" )

       self.assertListEqual(tokenizer.tokenize("" )       ,						[] )

       self.assertListEqual(tokenizer.tokenize("こんにちは" )       ,						["こ", "ん", "に", "ち", "は"] )

       self.assertListEqual(tokenizer.tokenize("こんにちほ" )       ,						["こ", "ん", "に", "ち", "[UNK]"] )




    def      _a    (    self ):
       lowerCAmelCase_:							Optional[Any]       =	self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char" )

       lowerCAmelCase_:							Any       =	tokenizer.encode("ありがとう。"       ,						add_special_tokens=lowerCamelCase__ )
       lowerCAmelCase_:							Dict       =	tokenizer.encode("どういたしまして。"       ,						add_special_tokens=lowerCamelCase__ )

       lowerCAmelCase_:							Dict       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__ )
       lowerCAmelCase_:							Any       =	tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,						lowerCamelCase__ )

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Union[str, Any]       =	"cl-tohoku/bert-base-japanese"
       lowerCAmelCase_:							Optional[int]       =	AutoTokenizer.from_pretrained(lowerCamelCase__ )
       self.assertIsInstance(lowerCamelCase__       ,						lowerCamelCase__ )





class 					_lowercase    (     unittest.TestCase			):






    '''simple docstring'''




    def      _a    (    self ):
       lowerCAmelCase_:							Dict       =	"cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )
       lowerCAmelCase_:							List[str]       =	"bert-base-cased"
       with self.assertLogs("transformers"       ,						level="WARNING" ) as cm:
          BertJapaneseTokenizer.from_pretrained(lowerCamelCase__ )
          self.assertTrue(
              cm.records[0].message.startswith(
                  "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                  " is called from." ) )