





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFConvBertForMaskedLM,
         TFConvBertForMultipleChoice,
         TFConvBertForQuestionAnswering,
         TFConvBertForSequenceClassification,
         TFConvBertForTokenClassification,
         TFConvBertModel,
     )



class 				_UpperCAmelCase    :

  """simple docstring"""


  def __init__(				self :		Any					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		int=1_3					,  lowerCAmelCase_ :		Optional[Any]=7					,  lowerCAmelCase_ :		Any=True					,  lowerCAmelCase_ :		Optional[int]=True					,  lowerCAmelCase_ :		Union[str, Any]=True					,  lowerCAmelCase_ :		Dict=True					,  lowerCAmelCase_ :		Union[str, Any]=9_9					,  lowerCAmelCase_ :		List[str]=3_2					,  lowerCAmelCase_ :		int=2					,  lowerCAmelCase_ :		Tuple=4					,  lowerCAmelCase_ :		Any=3_7					,  lowerCAmelCase_ :		str="gelu"					,  lowerCAmelCase_ :		Optional[int]=0.1					,  lowerCAmelCase_ :		str=0.1					,  lowerCAmelCase_ :		Any=5_1_2					,  lowerCAmelCase_ :		Optional[Any]=1_6					,  lowerCAmelCase_ :		Optional[Any]=2					,  lowerCAmelCase_ :		str=0.02					,  lowerCAmelCase_ :		List[str]=3					,  lowerCAmelCase_ :		List[Any]=4					,  lowerCAmelCase_ :		int=None					,  )							->						Dict:
     __lowerCAmelCase								=	parent
     __lowerCAmelCase								=	1_3
     __lowerCAmelCase								=	7
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	9_9
     __lowerCAmelCase								=	3_8_4
     __lowerCAmelCase								=	2
     __lowerCAmelCase								=	4
     __lowerCAmelCase								=	3_7
     __lowerCAmelCase								=	'gelu'
     __lowerCAmelCase								=	0.1
     __lowerCAmelCase								=	0.1
     __lowerCAmelCase								=	5_1_2
     __lowerCAmelCase								=	1_6
     __lowerCAmelCase								=	2
     __lowerCAmelCase								=	0.02
     __lowerCAmelCase								=	3
     __lowerCAmelCase								=	4
     __lowerCAmelCase								=	1_2_8
     __lowerCAmelCase								=	2
     __lowerCAmelCase								=	9
     __lowerCAmelCase								=	1
     __lowerCAmelCase								=	None






  def 				lowercase		(				self :		List[Any]     )							->						str:
     __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.vocab_size     )

     __lowerCAmelCase								=	None
     if self.use_input_mask:
        __lowerCAmelCase								=	random_attention_mask([self.batch_size, self.seq_length]     )

     __lowerCAmelCase								=	None
     if self.use_token_type_ids:
        __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.type_vocab_size     )

     __lowerCAmelCase								=	None
     __lowerCAmelCase								=	None
     __lowerCAmelCase								=	None
     if self.use_labels:
        __lowerCAmelCase								=	ids_tensor([self.batch_size]					,  self.type_sequence_label_size     )
        __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length]					,  self.num_labels     )
        __lowerCAmelCase								=	ids_tensor([self.batch_size]					,  self.num_choices     )

     __lowerCAmelCase								=	ConvBertConfig(
         vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  intermediate_size=self.intermediate_size					,  hidden_act=self.hidden_act					,  hidden_dropout_prob=self.hidden_dropout_prob					,  attention_probs_dropout_prob=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  type_vocab_size=self.type_vocab_size					,  initializer_range=self.initializer_range					,  return_dict=lowerCAmelCase_					,  )

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		str     )							->						Dict:
     __lowerCAmelCase								=	TFConvBertModel(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

     __lowerCAmelCase								=	[input_ids, input_mask]
     __lowerCAmelCase								=	model(lowerCAmelCase_     )

     __lowerCAmelCase								=	model(lowerCAmelCase_     )

     self.parent.assertEqual(result.last_hidden_state.shape					,  (self.batch_size, self.seq_length, self.hidden_size)     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		Union[str, Any]     )							->						Dict:
     __lowerCAmelCase								=	TFConvBertForMaskedLM(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	{
         'input_ids': input_ids,
         'attention_mask': input_mask,
         'token_type_ids': token_type_ids,
     }
     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.vocab_size)     )






  def 				lowercase		(				self :		List[str]					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		int					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Union[str, Any]					,  lowerCAmelCase_ :		int     )							->						List[Any]:
     __lowerCAmelCase								=	self.num_labels
     __lowerCAmelCase								=	TFConvBertForSequenceClassification(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	{
         'input_ids': input_ids,
         'attention_mask': input_mask,
         'token_type_ids': token_type_ids,
     }

     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_labels)     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		Any					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		List[Any]     )							->						str:
     __lowerCAmelCase								=	self.num_choices
     __lowerCAmelCase								=	TFConvBertForMultipleChoice(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	tf.tile(tf.expand_dims(lowerCAmelCase_					,  1     )					,  (1, self.num_choices, 1)     )
     __lowerCAmelCase								=	tf.tile(tf.expand_dims(lowerCAmelCase_					,  1     )					,  (1, self.num_choices, 1)     )
     __lowerCAmelCase								=	tf.tile(tf.expand_dims(lowerCAmelCase_					,  1     )					,  (1, self.num_choices, 1)     )
     __lowerCAmelCase								=	{
         'input_ids': multiple_choice_inputs_ids,
         'attention_mask': multiple_choice_input_mask,
         'token_type_ids': multiple_choice_token_type_ids,
     }
     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.num_choices)     )






  def 				lowercase		(				self :		Dict					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		Tuple     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.num_labels
     __lowerCAmelCase								=	TFConvBertForTokenClassification(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	{
         'input_ids': input_ids,
         'attention_mask': input_mask,
         'token_type_ids': token_type_ids,
     }
     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.logits.shape					,  (self.batch_size, self.seq_length, self.num_labels)     )






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Optional[int]					,  lowerCAmelCase_ :		Dict					,  lowerCAmelCase_ :		List[Any]					,  lowerCAmelCase_ :		Optional[int]     )							->						Any:
     __lowerCAmelCase								=	TFConvBertForQuestionAnswering(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	{
         'input_ids': input_ids,
         'attention_mask': input_mask,
         'token_type_ids': token_type_ids,
     }

     __lowerCAmelCase								=	model(lowerCAmelCase_     )
     self.parent.assertEqual(result.start_logits.shape					,  (self.batch_size, self.seq_length)     )
     self.parent.assertEqual(result.end_logits.shape					,  (self.batch_size, self.seq_length)     )






  def 				lowercase		(				self :		List[str]     )							->						str:
     __lowerCAmelCase								=	self.prepare_config_and_inputs()
     (
         (
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	(
         __lowerCAmelCase
     )	,	
     )								=	config_and_inputs
     __lowerCAmelCase								=	{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict





@require_tf
class 				_UpperCAmelCase    (    _UpperCamelCase		,    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = (
      (
          TFConvBertModel,
          TFConvBertForMaskedLM,
          TFConvBertForQuestionAnswering,
          TFConvBertForSequenceClassification,
          TFConvBertForTokenClassification,
          TFConvBertForMultipleChoice,
      )
      if is_tf_available()
      else ()
  )
  a_        = (
      {
          """feature-extraction""": TFConvBertModel,
          """fill-mask""": TFConvBertForMaskedLM,
          """question-answering""": TFConvBertForQuestionAnswering,
          """text-classification""": TFConvBertForSequenceClassification,
          """token-classification""": TFConvBertForTokenClassification,
          """zero-shot""": TFConvBertForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  a_        = False
  a_        = False
  a_        = False






  def 				lowercase		(				self :		Union[str, Any]     )							->						List[str]:
     __lowerCAmelCase								=	TFConvBertModelTester(self     )
     __lowerCAmelCase								=	ConfigTester(self					,  config_class=lowerCAmelCase_					,  hidden_size=3_7     )






  def 				lowercase		(				self :		int     )							->						Optional[Any]:
     self.config_tester.run_common_tests()






  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*lowerCAmelCase_     )






  def 				lowercase		(				self :		List[str]     )							->						Dict:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[int]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_     )






  def 				lowercase		(				self :		int     )							->						Tuple:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_     )






  def 				lowercase		(				self :		str     )							->						Tuple:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_     )






  @slow
  def 				lowercase		(				self :		Optional[int]     )							->						Tuple:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs_for_common()
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	True

     if hasattr(lowerCAmelCase_					,  'use_cache'     ):
        __lowerCAmelCase								=	True

     __lowerCAmelCase								=	getattr(self.model_tester					,  'encoder_seq_length'					,  self.model_tester.seq_length     )
     __lowerCAmelCase								=	getattr(self.model_tester					,  'key_length'					,  lowerCAmelCase_     )

     for model_class in self.all_model_classes:
        __lowerCAmelCase								=	self._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_     )
        __lowerCAmelCase								=	model_class(lowerCAmelCase_     )
        __lowerCAmelCase								=	len(model(lowerCAmelCase_     )     )

        with tempfile.TemporaryDirectory() as tmpdirname:
           model.save_pretrained(lowerCAmelCase_					,  saved_model=lowerCAmelCase_     )
           __lowerCAmelCase								=	os.path.join(lowerCAmelCase_					,  'saved_model'					,  '1'     )
           __lowerCAmelCase								=	tf.keras.models.load_model(lowerCAmelCase_     )
           __lowerCAmelCase								=	model(lowerCAmelCase_     )

           if self.is_encoder_decoder:
              __lowerCAmelCase								=	outputs['encoder_hidden_states']
              __lowerCAmelCase								=	outputs['encoder_attentions']
           else:
              __lowerCAmelCase								=	outputs['hidden_states']
              __lowerCAmelCase								=	outputs['attentions']

           self.assertEqual(len(lowerCAmelCase_     )					,  lowerCAmelCase_     )

           __lowerCAmelCase								=	getattr(
               self.model_tester					,  'expected_num_hidden_layers'					,  self.model_tester.num_hidden_layers + 1     )

           self.assertEqual(len(lowerCAmelCase_     )					,  lowerCAmelCase_     )
           self.assertListEqual(
               list(output_hidden_states[0].shape[-2:]     )					,  [self.model_tester.seq_length, self.model_tester.hidden_size]					,  )

           self.assertEqual(len(lowerCAmelCase_     )					,  self.model_tester.num_hidden_layers     )
           self.assertListEqual(
               list(output_attentions[0].shape[-3:]     )					,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,  )






  @slow
  def 				lowercase		(				self :		int     )							->						Optional[Any]:
     __lowerCAmelCase								=	TFConvBertModel.from_pretrained('YituTech/conv-bert-base'     )
     self.assertIsNotNone(lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple     )							->						Optional[int]:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs_for_common()
     __lowerCAmelCase								=	True
     __lowerCAmelCase								=	getattr(self.model_tester					,  'decoder_seq_length'					,  self.model_tester.seq_length     )
     __lowerCAmelCase								=	getattr(self.model_tester					,  'encoder_seq_length'					,  self.model_tester.seq_length     )
     __lowerCAmelCase								=	getattr(self.model_tester					,  'key_length'					,  lowerCAmelCase_     )
     __lowerCAmelCase								=	getattr(self.model_tester					,  'key_length'					,  lowerCAmelCase_     )

     def check_decoder_attentions_output(lowerCAmelCase_ :		List[str]     ):
        __lowerCAmelCase								=	len(lowerCAmelCase_     )
        self.assertEqual(out_len % 2					,  0     )
        __lowerCAmelCase								=	outputs.decoder_attentions
        self.assertEqual(len(lowerCAmelCase_     )					,  self.model_tester.num_hidden_layers     )
        self.assertListEqual(
            list(decoder_attentions[0].shape[-3:]     )					,  [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]					,  )

     def check_encoder_attentions_output(lowerCAmelCase_ :		Tuple     ):
        __lowerCAmelCase								=	[
            t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
        ]
        self.assertEqual(len(lowerCAmelCase_     )					,  self.model_tester.num_hidden_layers     )
        self.assertListEqual(
            list(attentions[0].shape[-3:]     )					,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,  )

     for model_class in self.all_model_classes:
        __lowerCAmelCase								=	True
        __lowerCAmelCase								=	False
        __lowerCAmelCase								=	model_class(lowerCAmelCase_     )
        __lowerCAmelCase								=	model(self._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_     )     )
        __lowerCAmelCase								=	len(lowerCAmelCase_     )
        self.assertEqual(config.output_hidden_states					,  lowerCAmelCase_     )
        check_encoder_attentions_output(lowerCAmelCase_     )

        if self.is_encoder_decoder:
           __lowerCAmelCase								=	model_class(lowerCAmelCase_     )
           __lowerCAmelCase								=	model(self._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_     )     )
           self.assertEqual(config.output_hidden_states					,  lowerCAmelCase_     )
           check_decoder_attentions_output(lowerCAmelCase_     )

        # Check that output attentions can also be changed via the config
        del inputs_dict["output_attentions"]
        __lowerCAmelCase								=	True
        __lowerCAmelCase								=	model_class(lowerCAmelCase_     )
        __lowerCAmelCase								=	model(self._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_     )     )
        self.assertEqual(config.output_hidden_states					,  lowerCAmelCase_     )
        check_encoder_attentions_output(lowerCAmelCase_     )

        # Check attention is always last and order is fine
        __lowerCAmelCase								=	True
        __lowerCAmelCase								=	True
        __lowerCAmelCase								=	model_class(lowerCAmelCase_     )
        __lowerCAmelCase								=	model(self._prepare_for_class(lowerCAmelCase_					,  lowerCAmelCase_     )     )

        self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)					,  len(lowerCAmelCase_     )     )
        self.assertEqual(model.config.output_hidden_states					,  lowerCAmelCase_     )
        check_encoder_attentions_output(lowerCAmelCase_     )





@require_tf
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  @slow
  def 				lowercase		(				self :		str     )							->						Tuple:
     __lowerCAmelCase								=	TFConvBertModel.from_pretrained('YituTech/conv-bert-base'     )
     __lowerCAmelCase								=	tf.constant([[0, 1, 2, 3, 4, 5]]     )
     __lowerCAmelCase								=	model(lowerCAmelCase_     )[0]

     __lowerCAmelCase								=	[1, 6, 7_6_8]
     self.assertEqual(output.shape					,  lowerCAmelCase_     )

     __lowerCAmelCase								=	tf.constant(
         [
             [
                 [-0.03_47_54_93, -0.4_68_60_34, -0.30_63_88_32],
                 [0.22_63_72_48, -0.26_98_86_46, -0.7_42_34_24],
                 [0.10_32_48_68, -0.45_01_35_08, -0.58_28_07_84],
             ]
         ]     )
     tf.debugging.assert_near(output[:, :3, :3]					,  lowerCAmelCase_					,  atol=1e-4     )








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False
  a_        = True






  def 				lowercase		(				self :		Optional[Any]     )							->						List[str]:
     super().setUp()

     __lowerCAmelCase								=	[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple     )							->						str:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		str     )							->						Dict:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.get_input_output_texts(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode(lowerCAmelCase_					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(lowerCAmelCase_					,  clean_up_tokenization_spaces=lowerCAmelCase_     )
     return text, ids






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Optional[Any]     )							->						Optional[Any]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )






  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='mecab'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	MecabTokenizer(mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		List[Any]     )							->						int:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic_lite'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(do_lower_case=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(
            do_lower_case=lowerCAmelCase_					,  normalize_text=lowerCAmelCase_					,  mecab_option='-d /usr/local/lib/mecab/dic/jumandic'     )
     except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(normalize_text=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,  )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='sudachi'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='A'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国', '人', '参政', '権']     )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='B'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人', '参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='C'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		Dict     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(do_lower_case=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(normalize_text=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	SudachiTokenizer(trim_whitespace=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='jumanpp'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_jumanpp
  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	JumanppTokenizer(do_lower_case=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Dict     )							->						Dict:
     __lowerCAmelCase								=	JumanppTokenizer(normalize_text=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	JumanppTokenizer(trim_whitespace=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Any:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'     )					,  ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						str:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	WordpieceTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こんにちは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは'     )					,  ['こん', '##ばんは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'     )					,  ['こん', '##ばんは', '[UNK]', 'こんにちは']     )






  def 				lowercase		(				self :		List[Any]     )							->						Tuple:
     __lowerCAmelCase								=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'     )
     __lowerCAmelCase								=	tokenizer.subword_tokenizer

     __lowerCAmelCase								=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']     )

     __lowerCAmelCase								=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False






  def 				lowercase		(				self :		Optional[Any]     )							->						Tuple:
     super().setUp()

     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		str					,  **lowerCAmelCase_ :		Tuple     )							->						Union[str, Any]:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,  subword_tokenizer_type='character'					,  **lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Tuple     )							->						Optional[int]:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text






  def 				lowercase		(				self :		Dict     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Any     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		List[Any]     )							->						int:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		str     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  subword_tokenizer_type='character'     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'     )
     self.assertListEqual(
         lowerCAmelCase_					,  ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']     )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]     )






  def 				lowercase		(				self :		str     )							->						Optional[int]:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	CharacterTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こ', 'ん', 'に', 'ち', 'は']     )

     self.assertListEqual(tokenizer.tokenize('こんにちほ'     )					,  ['こ', 'ん', 'に', 'ち', '[UNK]']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						Union[str, Any]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_     )
     self.assertIsInstance(lowerCAmelCase_					,  lowerCAmelCase_     )





class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )
     __lowerCAmelCase								=	'bert-base-cased'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertJapaneseTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )

