





import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    WavaVecaConformerConfig,
    WavaVecaConformerForCTC,
    WavaVecaConformerForPreTraining,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
_snake_case  :    Optional[Any]						    =  logging.get_logger(__name__)

_snake_case  :    Optional[int]						    =  {
    'post_extract_proj': 'feature_projection.projection',
    'encoder.pos_conv.0': 'encoder.pos_conv_embed.conv',
    'self_attn.linear_k': 'encoder.layers.*.self_attn.linear_k',
    'self_attn.linear_v': 'encoder.layers.*.self_attn.linear_v',
    'self_attn.linear_q': 'encoder.layers.*.self_attn.linear_q',
    'self_attn.pos_bias_u': 'encoder.layers.*.self_attn.pos_bias_u',
    'self_attn.pos_bias_v': 'encoder.layers.*.self_attn.pos_bias_v',
    'self_attn.linear_out': 'encoder.layers.*.self_attn.linear_out',
    'self_attn.linear_pos': 'encoder.layers.*.self_attn.linear_pos',
    'self_attn.rotary_emb': 'encoder.embed_positions',
    'self_attn_layer_norm': 'encoder.layers.*.self_attn_layer_norm',
    'conv_module.pointwise_conv1': 'encoder.layers.*.conv_module.pointwise_conv1',
    'conv_module.pointwise_conv2': 'encoder.layers.*.conv_module.pointwise_conv2',
    'conv_module.depthwise_conv': 'encoder.layers.*.conv_module.depthwise_conv',
    'conv_module.batch_norm': 'encoder.layers.*.conv_module.batch_norm',
    'conv_module.layer_norm': 'encoder.layers.*.conv_module.layer_norm',
    'ffn1.w_1': 'encoder.layers.*.ffn1.intermediate_dense',
    'ffn1.w_2': 'encoder.layers.*.ffn1.output_dense',
    'ffn1.layer_norm': 'encoder.layers.*.ffn1_layer_norm',
    'ffn2.w_1': 'encoder.layers.*.ffn2.intermediate_dense',
    'ffn2.w_2': 'encoder.layers.*.ffn2.output_dense',
    'ffn2.layer_norm': 'encoder.layers.*.ffn2_layer_norm',
    'final_layer_norm': 'encoder.layers.*.final_layer_norm',
    'encoder.layer_norm': 'encoder.layer_norm',
    'w2v_model.layer_norm': 'feature_projection.layer_norm',
    'quantizer.weight_proj': 'quantizer.weight_proj',
    'quantizer.vars': 'quantizer.codevectors',
    'project_q': 'project_q',
    'final_proj': 'project_hid',
    'w2v_encoder.proj': 'lm_head',
    'mask_emb': 'masked_spec_embed',
}
_snake_case  :    Union[str, Any]						    =  [
    'lm_head',
    'quantizer.weight_proj',
    'quantizer.codevectors',
    'project_q',
    'project_hid',
]

def   a_    ( lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			Tuple,	lowerCAmelCase_      :			List[str],	lowerCAmelCase_      :			List[Any],	lowerCAmelCase_      :			int	):
   for attribute in key.split('.'	):
      __lowerCAmelCase								=	getattr(lowerCAmelCase_,	lowerCAmelCase_	)

   if weight_type is not None:
      __lowerCAmelCase								=	getattr(lowerCAmelCase_,	lowerCAmelCase_	).shape
   else:
      __lowerCAmelCase								=	hf_pointer.shape

   if hf_shape != value.shape:
      raise ValueError(
          F"""Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be"""
          F""" {value.shape} for {full_name}"""	)

   if weight_type == "weight":
      __lowerCAmelCase								=	value
   elif weight_type == "weight_g":
      __lowerCAmelCase								=	value
   elif weight_type == "weight_v":
      __lowerCAmelCase								=	value
   elif weight_type == "bias":
      __lowerCAmelCase								=	value
   elif weight_type == "running_mean":
      __lowerCAmelCase								=	value
   elif weight_type == "running_var":
      __lowerCAmelCase								=	value
   elif weight_type == "num_batches_tracked":
      __lowerCAmelCase								=	value
   elif weight_type == "inv_freq":
      __lowerCAmelCase								=	value
   else:
      __lowerCAmelCase								=	value

   logger.info(F"""{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}."""	)

def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Optional[Any]	):
   __lowerCAmelCase								=	[]
   __lowerCAmelCase								=	fairseq_model.state_dict()

   __lowerCAmelCase								=	hf_model.wavaveca_conformer.feature_extractor

   for name, value in fairseq_dict.items():
      __lowerCAmelCase								=	False
      if "conv_layers" in name:
         load_conv_layer(
             lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	hf_model.config.feat_extract_norm == 'group',	)
         __lowerCAmelCase								=	True
      else:
         for key, mapped_key in MAPPING.items():
            __lowerCAmelCase								=	'wav2vec2_conformer.' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
            if key in name or key.split('w2v_model.'	)[-1] == name.split('.'	)[0]:
               __lowerCAmelCase								=	True
               if "*" in mapped_key:
                  __lowerCAmelCase								=	name.split(lowerCAmelCase_	)[0].split('.'	)[-2]
                  __lowerCAmelCase								=	mapped_key.replace('*',	lowerCAmelCase_	)
               if "pos_bias_u" in name:
                  __lowerCAmelCase								=	None
               elif "pos_bias_v" in name:
                  __lowerCAmelCase								=	None
               elif "weight_g" in name:
                  __lowerCAmelCase								=	'weight_g'
               elif "weight_v" in name:
                  __lowerCAmelCase								=	'weight_v'
               elif "bias" in name:
                  __lowerCAmelCase								=	'bias'
               elif "weight" in name:
                  # TODO: don't match quantizer.weight_proj
                  __lowerCAmelCase								=	'weight'
               elif "running_mean" in name:
                  __lowerCAmelCase								=	'running_mean'
               elif "inv_freq" in name:
                  __lowerCAmelCase								=	'inv_freq'
               elif "running_var" in name:
                  __lowerCAmelCase								=	'running_var'
               elif "num_batches_tracked" in name:
                  __lowerCAmelCase								=	'num_batches_tracked'
               else:
                  __lowerCAmelCase								=	None
               set_recursively(lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)
            continue
      if not is_used:
         unused_weights.append(lowerCAmelCase_	)

   logger.warning(F"""Unused weights: {unused_weights}"""	)

def   a_    ( lowerCAmelCase_      :			Tuple,	lowerCAmelCase_      :			Optional[int],	lowerCAmelCase_      :			Dict,	lowerCAmelCase_      :			List[str],	lowerCAmelCase_      :			Dict	):
   __lowerCAmelCase								=	full_name.split('conv_layers.'	)[-1]
   __lowerCAmelCase								=	name.split('.'	)
   __lowerCAmelCase								=	int(items[0]	)
   __lowerCAmelCase								=	int(items[1]	)

   if type_id == 0:
      if "bias" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""	)
         __lowerCAmelCase								=	value
         logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""	)
      elif "weight" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""	)
         __lowerCAmelCase								=	value
         logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""	)
   elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
      if "bias" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found."""	)
         __lowerCAmelCase								=	value
         logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""	)
      elif "weight" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found."""	)
         __lowerCAmelCase								=	value
         logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""	)
   else:
      unused_weights.append(lowerCAmelCase_	)

@torch.no_grad()
def   a_    ( lowerCAmelCase_      :			Union[str, Any],	lowerCAmelCase_      :			int,	lowerCAmelCase_      :			Tuple=None,	lowerCAmelCase_      :			List[Any]=None,	lowerCAmelCase_      :			Union[str, Any]=True	):
   if config_path is not None:
      __lowerCAmelCase								=	WavaVecaConformerConfig.from_pretrained(lowerCAmelCase_,	hidden_act='swish'	)
   else:
      __lowerCAmelCase								=	WavaVecaConformerConfig()

   if "rope" in checkpoint_path:
      __lowerCAmelCase								=	'rotary'

   if is_finetuned:
      if dict_path:
         __lowerCAmelCase								=	Dictionary.load(lowerCAmelCase_	)

         # important change bos & pad token id since CTC symbol is <pad> and
         # not <s> as in fairseq
         __lowerCAmelCase								=	target_dict.pad_index
         __lowerCAmelCase								=	target_dict.bos_index
         __lowerCAmelCase								=	target_dict.eos_index
         __lowerCAmelCase								=	len(target_dict.symbols	)
         __lowerCAmelCase								=	os.path.join(lowerCAmelCase_,	'vocab.json'	)
         if not os.path.isdir(lowerCAmelCase_	):
            logger.error('--pytorch_dump_folder_path ({}) should be a directory'.format(lowerCAmelCase_	)	)
            return
         os.makedirs(lowerCAmelCase_,	exist_ok=lowerCAmelCase_	)
         __lowerCAmelCase								=	target_dict.indices

         # fairseq has the <pad> and <s> switched
         __lowerCAmelCase								=	0
         __lowerCAmelCase								=	1
         with open(lowerCAmelCase_,	'w',	encoding='utf-8'	) as vocab_handle:
            json.dump(lowerCAmelCase_,	lowerCAmelCase_	)
         __lowerCAmelCase								=	WavaVecaCTCTokenizer(
             lowerCAmelCase_,	unk_token=target_dict.unk_word,	pad_token=target_dict.pad_word,	bos_token=target_dict.bos_word,	eos_token=target_dict.eos_word,	word_delimiter_token='|',	do_lower_case=lowerCAmelCase_,	)
         __lowerCAmelCase								=	True if config.feat_extract_norm == 'layer' else False
         __lowerCAmelCase								=	WavaVecaFeatureExtractor(
             feature_size=1,	sampling_rate=1_6000,	padding_value=0,	do_normalize=lowerCAmelCase_,	return_attention_mask=lowerCAmelCase_,	)
         __lowerCAmelCase								=	WavaVecaProcessor(feature_extractor=lowerCAmelCase_,	tokenizer=lowerCAmelCase_	)
         processor.save_pretrained(lowerCAmelCase_	)

      __lowerCAmelCase								=	WavaVecaConformerForCTC(lowerCAmelCase_	)
   else:
      __lowerCAmelCase								=	WavaVecaConformerForPreTraining(lowerCAmelCase_	)

   if is_finetuned:
      __lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase								=	fairseq.checkpoint_utils.load_model_ensemble_and_task(
          [checkpoint_path],	arg_overrides={'data': '/'.join(dict_path.split('/'	)[:-1]	)}	)
   else:
      __lowerCAmelCase								=	argparse.Namespace(task='audio_pretraining'	)
      __lowerCAmelCase								=	fairseq.tasks.setup_task(lowerCAmelCase_	)

      __lowerCAmelCase	,	__lowerCAmelCase	,	__lowerCAmelCase								=	fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path],	task=lowerCAmelCase_	)

   __lowerCAmelCase								=	model[0].eval()

   recursively_load_weights(lowerCAmelCase_,	lowerCAmelCase_,	not is_finetuned	)

   hf_wavavec.save_pretrained(lowerCAmelCase_	)


if __name__ == "__main__":
     _snake_case  :    Dict						    =  argparse.ArgumentParser()
     parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
     parser.add_argument('--checkpoint_path', default=None, type=str, help='Path to fairseq checkpoint')
     parser.add_argument('--dict_path', default=None, type=str, help='Path to dict of fine-tuned model')
     parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
     parser.add_argument(
         '--not_finetuned', action='store_true', help='Whether the model to convert is a fine-tuned model or not'
     )
     _snake_case  :    List[Any]						    =  parser.parse_args()
     convert_wavaveca_conformer_checkpoint(
         args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
     )








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False
  a_        = True






  def 				lowercase		(				self :		Optional[Any]     )							->						List[str]:
     super().setUp()

     __lowerCAmelCase								=	[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         'こんにちは',
         'こん',
         'にちは',
         'ばんは',
         '##こん',
         '##にちは',
         '##ばんは',
         '世界',
         '##世界',
         '、',
         '##、',
         '。',
         '##。',
     ]

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		Tuple     )							->						str:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こんにちは 、 世界 。 こんばんは 、 世界 。'
     return input_text, output_text






  def 				lowercase		(				self :		List[Any]					,  lowerCAmelCase_ :		str     )							->						Dict:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.get_input_output_texts(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode(lowerCAmelCase_					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(lowerCAmelCase_					,  clean_up_tokenization_spaces=lowerCAmelCase_     )
     return text, ids






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Optional[Any]     )							->						Optional[Any]:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )






  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='mecab'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		Dict     )							->						Tuple:
     __lowerCAmelCase								=	MecabTokenizer(mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		List[Any]     )							->						int:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic_lite'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(mecab_dic='unidic'     )
     except ModuleNotFoundError:
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Tuple     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(do_lower_case=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Optional[Any]:
     try:
        __lowerCAmelCase								=	MecabTokenizer(
            do_lower_case=lowerCAmelCase_					,  normalize_text=lowerCAmelCase_					,  mecab_option='-d /usr/local/lib/mecab/dic/jumandic'     )
     except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	MecabTokenizer(normalize_text=lowerCAmelCase_					,  mecab_dic='ipadic'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,  )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='sudachi'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='A'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国', '人', '参政', '権']     )






  @require_sudachi
  def 				lowercase		(				self :		Tuple     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='B'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人', '参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		List[str]     )							->						Union[str, Any]:
     __lowerCAmelCase								=	SudachiTokenizer(sudachi_dict_type='core'					,  sudachi_split_mode='C'     )

     self.assertListEqual(tokenizer.tokenize('外国人参政権'     )					,  ['外国人参政権']     )






  @require_sudachi
  def 				lowercase		(				self :		Dict     )							->						List[str]:
     __lowerCAmelCase								=	SudachiTokenizer(do_lower_case=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		Union[str, Any]     )							->						List[Any]:
     __lowerCAmelCase								=	SudachiTokenizer(normalize_text=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,  )






  @require_sudachi
  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	SudachiTokenizer(trim_whitespace=lowerCAmelCase_					,  sudachi_dict_type='core'     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Union[str, Any]     )							->						Any:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  word_tokenizer_type='jumanpp'     )
     self.assertIsNotNone(lowerCAmelCase_     )

     __lowerCAmelCase								=	'こんにちは、世界。\nこんばんは、世界。'
     __lowerCAmelCase								=	tokenizer.tokenize(lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]     )

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  'tokenizer.bin'     )
     with open(lowerCAmelCase_					,  'wb'     ) as handle:
        pickle.dump(lowerCAmelCase_					,  lowerCAmelCase_     )

     with open(lowerCAmelCase_					,  'rb'     ) as handle:
        __lowerCAmelCase								=	pickle.load(lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer_new.tokenize(lowerCAmelCase_     )

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  @require_jumanpp
  def 				lowercase		(				self :		List[Any]     )							->						Optional[Any]:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Union[str, Any]:
     __lowerCAmelCase								=	JumanppTokenizer(do_lower_case=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Dict     )							->						Dict:
     __lowerCAmelCase								=	JumanppTokenizer(normalize_text=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	JumanppTokenizer(trim_whitespace=lowerCAmelCase_     )

     self.assertListEqual(
         tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '     )					,  ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,  )






  @require_jumanpp
  def 				lowercase		(				self :		Any     )							->						Any:
     __lowerCAmelCase								=	JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'     )					,  ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,  )






  def 				lowercase		(				self :		Any     )							->						str:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	WordpieceTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こんにちは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは'     )					,  ['こん', '##ばんは']     )

     self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'     )					,  ['こん', '##ばんは', '[UNK]', 'こんにちは']     )






  def 				lowercase		(				self :		List[Any]     )							->						Tuple:
     __lowerCAmelCase								=	BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'     )
     __lowerCAmelCase								=	tokenizer.subword_tokenizer

     __lowerCAmelCase								=	subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']     )

     __lowerCAmelCase								=	subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'     )
     self.assertListEqual(lowerCAmelCase_					,  ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = BertJapaneseTokenizer
  a_        = False






  def 				lowercase		(				self :		Optional[Any]     )							->						Tuple:
     super().setUp()

     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file					,  'w'					,  encoding='utf-8'     ) as vocab_writer:
        vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )






  def 				lowercase		(				self :		str					,  **lowerCAmelCase_ :		Tuple     )							->						Union[str, Any]:
     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,  subword_tokenizer_type='character'					,  **lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Tuple     )							->						Optional[int]:
     __lowerCAmelCase								=	'こんにちは、世界。 \nこんばんは、世界。'
     __lowerCAmelCase								=	'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
     return input_text, output_text






  def 				lowercase		(				self :		Dict     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		Any     )							->						str:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		List[Any]     )							->						int:
     pass  # TODO add if relevant






  def 				lowercase		(				self :		str     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class(self.vocab_file					,  subword_tokenizer_type='character'     )

     __lowerCAmelCase								=	tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'     )
     self.assertListEqual(
         lowerCAmelCase_					,  ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']     )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )					,  [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]     )






  def 				lowercase		(				self :		str     )							->						Optional[int]:
     __lowerCAmelCase								=	['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

     __lowerCAmelCase								=	{}
     for i, token in enumerate(lowerCAmelCase_     ):
        __lowerCAmelCase								=	i
     __lowerCAmelCase								=	CharacterTokenizer(vocab=lowerCAmelCase_					,  unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )					,  []     )

     self.assertListEqual(tokenizer.tokenize('こんにちは'     )					,  ['こ', 'ん', 'に', 'ち', 'は']     )

     self.assertListEqual(tokenizer.tokenize('こんにちほ'     )					,  ['こ', 'ん', 'に', 'ち', '[UNK]']     )






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'     )

     __lowerCAmelCase								=	tokenizer.encode('ありがとう。'					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.encode('どういたしまして。'					,  add_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					,  lowerCAmelCase_     )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		str     )							->						Union[str, Any]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     __lowerCAmelCase								=	AutoTokenizer.from_pretrained(lowerCAmelCase_     )
     self.assertIsInstance(lowerCAmelCase_					,  lowerCAmelCase_     )





class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	'cl-tohoku/bert-base-japanese'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )
     __lowerCAmelCase								=	'bert-base-cased'
     with self.assertLogs('transformers'					,  level='WARNING'     ) as cm:
        BertJapaneseTokenizer.from_pretrained(lowerCAmelCase_     )
        self.assertTrue(
            cm.records[0].message.startswith(
                'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                ' is called from.'     )     )

