


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin



@custom_tokenizers
class 						_UpperCAmelCase      (     lowercase		,		unittest.TestCase							):
		lowerCamelCase_	:			Union[str, Any]       			=  BertJapaneseTokenizer
		lowerCamelCase_	:			Union[str, Any]       			=  False
		lowerCamelCase_	:			Union[str, Any]       			=  True






		def _snake_case		(						self				:     List[Any]):
				super().setUp()

				SCREAMING_SNAKE_CASE_				:Tuple				     =						[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "こんにちは",
				    "こん",
				    "にちは",
				    "ばんは",
				    "##こん",
				    "##にちは",
				    "##ばんは",
				    "世界",
				    "##世界",
				    "、",
				    "##、",
				    "。",
				    "##。",
				]

				SCREAMING_SNAKE_CASE_				:List[Any]				     =						os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["vocab_file"])
				with open(self.vocab_file   ,  "w"   ,  encoding="utf-8") as vocab_writer:
						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))






		def _snake_case		(						self				:     List[str]   ,  UpperCAmelCase				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:Dict				     =						"こんにちは、世界。 \nこんばんは、世界。"
				SCREAMING_SNAKE_CASE_				:List[str]				     =						"こんにちは 、 世界 。 こんばんは 、 世界 。"
				return input_text, output_text






		def _snake_case		(						self				:     Optional[Any]   ,  UpperCAmelCase				:     Any):
				SCREAMING_SNAKE_CASE_     ,				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						self.get_input_output_texts(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						tokenizer.encode(UpperCAmelCase   ,  add_special_tokens=UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						tokenizer.decode(UpperCAmelCase   ,  clean_up_tokenization_spaces=UpperCAmelCase)
				return text, ids






		def _snake_case		(						self				:     Union[str, Any]):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     Optional[Any]):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     Dict):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:List[str]				     =						self.tokenizer_class(self.vocab_file)

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。")
				self.assertListEqual(UpperCAmelCase   ,  ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase)   ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])






		def _snake_case		(						self				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:int				     =						self.tokenizer_class(self.vocab_file   ,  word_tokenizer_type="mecab")
				self.assertIsNotNone(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						"こんにちは、世界。\nこんばんは、世界。"
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						tokenizer.tokenize(UpperCAmelCase)
				self.assertListEqual(UpperCAmelCase   ,  ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase)   ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						os.path.join(self.tmpdirname   ,  "tokenizer.bin")
				with open(UpperCAmelCase   ,  "wb") as handle:
						pickle.dump(UpperCAmelCase   ,  UpperCAmelCase)

				with open(UpperCAmelCase   ,  "rb") as handle:
						SCREAMING_SNAKE_CASE_				:int				     =						pickle.load(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Any				     =						tokenizer_new.tokenize(UpperCAmelCase)

				self.assertListEqual(UpperCAmelCase   ,  UpperCAmelCase)






		def _snake_case		(						self				:     List[str]):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						MecabTokenizer(mecab_dic="ipadic")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,  )






		def _snake_case		(						self				:     Any):
				try:
						SCREAMING_SNAKE_CASE_				:Tuple				     =						MecabTokenizer(mecab_dic="unidic_lite")
				except ModuleNotFoundError:
						return

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,  )






		def _snake_case		(						self				:     List[str]):
				try:
						SCREAMING_SNAKE_CASE_				:Dict				     =						MecabTokenizer(mecab_dic="unidic")
				except ModuleNotFoundError:
						return

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,  )






		def _snake_case		(						self				:     str):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						MecabTokenizer(do_lower_case=UpperCAmelCase   ,  mecab_dic="ipadic")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,  )






		def _snake_case		(						self				:     Union[str, Any]):
				try:
						SCREAMING_SNAKE_CASE_				:int				     =						MecabTokenizer(
						    do_lower_case=UpperCAmelCase   ,  normalize_text=UpperCAmelCase   ,  mecab_option="-d /usr/local/lib/mecab/dic/jumandic")
				except RuntimeError:
						# if dict doesn't exist in the system, previous code raises this error.
						return

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]   ,  )






		def _snake_case		(						self				:     int):
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						MecabTokenizer(normalize_text=UpperCAmelCase   ,  mecab_dic="ipadic")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]   ,  )






		@require_sudachi
		def _snake_case		(						self				:     List[str]):
				SCREAMING_SNAKE_CASE_				:str				     =						self.tokenizer_class(self.vocab_file   ,  word_tokenizer_type="sudachi")
				self.assertIsNotNone(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Tuple				     =						"こんにちは、世界。\nこんばんは、世界。"
				SCREAMING_SNAKE_CASE_				:List[str]				     =						tokenizer.tokenize(UpperCAmelCase)
				self.assertListEqual(UpperCAmelCase   ,  ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase)   ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

				SCREAMING_SNAKE_CASE_				:Tuple				     =						os.path.join(self.tmpdirname   ,  "tokenizer.bin")
				with open(UpperCAmelCase   ,  "wb") as handle:
						pickle.dump(UpperCAmelCase   ,  UpperCAmelCase)

				with open(UpperCAmelCase   ,  "rb") as handle:
						SCREAMING_SNAKE_CASE_				:Tuple				     =						pickle.load(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Any				     =						tokenizer_new.tokenize(UpperCAmelCase)

				self.assertListEqual(UpperCAmelCase   ,  UpperCAmelCase)






		@require_sudachi
		def _snake_case		(						self				:     Optional[Any]):
				SCREAMING_SNAKE_CASE_				:str				     =						SudachiTokenizer(sudachi_dict_type="core")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]   ,  )






		@require_sudachi
		def _snake_case		(						self				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						SudachiTokenizer(sudachi_dict_type="core"   ,  sudachi_split_mode="A")

				self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,  ["外国", "人", "参政", "権"])






		@require_sudachi
		def _snake_case		(						self				:     Optional[Any]):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						SudachiTokenizer(sudachi_dict_type="core"   ,  sudachi_split_mode="B")

				self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,  ["外国人", "参政権"])






		@require_sudachi
		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:Any				     =						SudachiTokenizer(sudachi_dict_type="core"   ,  sudachi_split_mode="C")

				self.assertListEqual(tokenizer.tokenize("外国人参政権")   ,  ["外国人参政権"])






		@require_sudachi
		def _snake_case		(						self				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:str				     =						SudachiTokenizer(do_lower_case=UpperCAmelCase   ,  sudachi_dict_type="core")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]   ,  )






		@require_sudachi
		def _snake_case		(						self				:     Tuple):
				SCREAMING_SNAKE_CASE_				:str				     =						SudachiTokenizer(normalize_text=UpperCAmelCase   ,  sudachi_dict_type="core")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]   ,  )






		@require_sudachi
		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:int				     =						SudachiTokenizer(trim_whitespace=UpperCAmelCase   ,  sudachi_dict_type="core")

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]   ,  )






		@require_jumanpp
		def _snake_case		(						self				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:List[str]				     =						self.tokenizer_class(self.vocab_file   ,  word_tokenizer_type="jumanpp")
				self.assertIsNotNone(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						"こんにちは、世界。\nこんばんは、世界。"
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						tokenizer.tokenize(UpperCAmelCase)
				self.assertListEqual(UpperCAmelCase   ,  ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase)   ,  [3, 12, 10, 14, 4, 9, 12, 10, 14])

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(self.tmpdirname   ,  "tokenizer.bin")
				with open(UpperCAmelCase   ,  "wb") as handle:
						pickle.dump(UpperCAmelCase   ,  UpperCAmelCase)

				with open(UpperCAmelCase   ,  "rb") as handle:
						SCREAMING_SNAKE_CASE_				:Dict				     =						pickle.load(UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:List[str]				     =						tokenizer_new.tokenize(UpperCAmelCase)

				self.assertListEqual(UpperCAmelCase   ,  UpperCAmelCase)






		@require_jumanpp
		def _snake_case		(						self				:     str):
				SCREAMING_SNAKE_CASE_				:int				     =						JumanppTokenizer()

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,  )






		@require_jumanpp
		def _snake_case		(						self				:     Dict):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						JumanppTokenizer(do_lower_case=UpperCAmelCase)

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,  )






		@require_jumanpp
		def _snake_case		(						self				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						JumanppTokenizer(normalize_text=UpperCAmelCase)

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]   ,  )






		@require_jumanpp
		def _snake_case		(						self				:     List[Any]):
				SCREAMING_SNAKE_CASE_				:List[str]				     =						JumanppTokenizer(trim_whitespace=UpperCAmelCase)

				self.assertListEqual(
				    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ")   ,  ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]   ,  )






		@require_jumanpp
		def _snake_case		(						self				:     int):
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						JumanppTokenizer()

				self.assertListEqual(
				    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。")   ,  ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]   ,  )






		def _snake_case		(						self				:     List[str]):
				SCREAMING_SNAKE_CASE_				:Any				     =						["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

				SCREAMING_SNAKE_CASE_				:Any				     =						{}
				for i, token in enumerate(UpperCAmelCase):
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						i
				SCREAMING_SNAKE_CASE_				:int				     =						WordpieceTokenizer(vocab=UpperCAmelCase   ,  unk_token="[UNK]")

				self.assertListEqual(tokenizer.tokenize("")   ,  [])

				self.assertListEqual(tokenizer.tokenize("こんにちは")   ,  ["こんにちは"])

				self.assertListEqual(tokenizer.tokenize("こんばんは")   ,  ["こん", "##ばんは"])

				self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは")   ,  ["こん", "##ばんは", "[UNK]", "こんにちは"])






		def _snake_case		(						self				:     Optional[Any]):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp")
				SCREAMING_SNAKE_CASE_				:Any				     =						tokenizer.subword_tokenizer

				SCREAMING_SNAKE_CASE_				:Any				     =						subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。")
				self.assertListEqual(UpperCAmelCase   ,  ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"])

				SCREAMING_SNAKE_CASE_				:Tuple				     =						subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは")
				self.assertListEqual(UpperCAmelCase   ,  ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"])







		def _snake_case		(						self				:     Union[str, Any]):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese")

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						tokenizer.encode("ありがとう。"   ,  add_special_tokens=UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						tokenizer.encode("どういたしまして。"   ,  add_special_tokens=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Tuple				     =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase   ,  UpperCAmelCase)

				# 2 is for "[CLS]", 3 is for "[SEP]"
				assert encoded_sentence == [2] + text + [3]
				assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 						_UpperCAmelCase      (     lowercase		,		unittest.TestCase							):
		lowerCamelCase_	:			Any       			=  BertJapaneseTokenizer
		lowerCamelCase_	:			Tuple       			=  False






		def _snake_case		(						self				:     Optional[Any]):
				super().setUp()

				SCREAMING_SNAKE_CASE_				:str				     =						["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["vocab_file"])
				with open(self.vocab_file   ,  "w"   ,  encoding="utf-8") as vocab_writer:
						vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))






		def _snake_case		(						self				:     int   ,  **UpperCAmelCase				:     Any):
				return BertJapaneseTokenizer.from_pretrained(self.tmpdirname   ,  subword_tokenizer_type="character"   ,  **UpperCAmelCase)






		def _snake_case		(						self				:     Union[str, Any]   ,  UpperCAmelCase				:     Optional[int]):
				SCREAMING_SNAKE_CASE_				:int				     =						"こんにちは、世界。 \nこんばんは、世界。"
				SCREAMING_SNAKE_CASE_				:List[str]				     =						"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
				return input_text, output_text






		def _snake_case		(						self				:     List[Any]):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     Optional[Any]):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     List[str]):
				pass  # TODO add if relevant






		def _snake_case		(						self				:     Optional[Any]):
				SCREAMING_SNAKE_CASE_				:str				     =						self.tokenizer_class(self.vocab_file   ,  subword_tokenizer_type="character")

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。")
				self.assertListEqual(
				    UpperCAmelCase   ,  ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"])
				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(UpperCAmelCase)   ,  [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])






		def _snake_case		(						self				:     List[Any]):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

				SCREAMING_SNAKE_CASE_				:List[str]				     =						{}
				for i, token in enumerate(UpperCAmelCase):
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						i
				SCREAMING_SNAKE_CASE_				:int				     =						CharacterTokenizer(vocab=UpperCAmelCase   ,  unk_token="[UNK]")

				self.assertListEqual(tokenizer.tokenize("")   ,  [])

				self.assertListEqual(tokenizer.tokenize("こんにちは")   ,  ["こ", "ん", "に", "ち", "は"])

				self.assertListEqual(tokenizer.tokenize("こんにちほ")   ,  ["こ", "ん", "に", "ち", "[UNK]"])







		def _snake_case		(						self				:     int):
				SCREAMING_SNAKE_CASE_				:Tuple				     =						self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char")

				SCREAMING_SNAKE_CASE_				:Any				     =						tokenizer.encode("ありがとう。"   ,  add_special_tokens=UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						tokenizer.encode("どういたしまして。"   ,  add_special_tokens=UpperCAmelCase)

				SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase)
				SCREAMING_SNAKE_CASE_				:List[str]				     =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase   ,  UpperCAmelCase)

				# 2 is for "[CLS]", 3 is for "[SEP]"
				assert encoded_sentence == [2] + text + [3]
				assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 						_UpperCAmelCase      (     unittest.TestCase							):






		def _snake_case		(						self				:     Any):
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						"cl-tohoku/bert-base-japanese"
				SCREAMING_SNAKE_CASE_				:Any				     =						AutoTokenizer.from_pretrained(UpperCAmelCase)
				self.assertIsInstance(UpperCAmelCase   ,  UpperCAmelCase)




class 						_UpperCAmelCase      (     unittest.TestCase							):






		def _snake_case		(						self				:     Tuple):
				SCREAMING_SNAKE_CASE_				:Optional[int]				     =						"cl-tohoku/bert-base-japanese"
				with self.assertLogs("transformers"   ,  level="WARNING") as cm:
						BertTokenizer.from_pretrained(UpperCAmelCase)
						self.assertTrue(
						    cm.records[0].message.startswith(
						        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
						        " is called from."))
				SCREAMING_SNAKE_CASE_				:Tuple				     =						"bert-base-cased"
				with self.assertLogs("transformers"   ,  level="WARNING") as cm:
						BertJapaneseTokenizer.from_pretrained(UpperCAmelCase)
						self.assertTrue(
						    cm.records[0].message.startswith(
						        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
						        " is called from."))










import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
		import torch.distributed.checkpoint as dist_cp
		from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
		from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
		from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
		from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


SCREAMING_SNAKE_CASE__     					=			get_logger(__name__)


def     lowercase	(    a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		os.makedirs(a     , exist_ok=a )
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						model.state_dict()
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Optional[int]				     =						F"{MODEL_NAME}.bin" if model_index == 0 else F"{MODEL_NAME}_{model_index}.bin"
						SCREAMING_SNAKE_CASE_				:str				     =						os.path.join(a     , a )
						if accelerator.process_index == 0:
								logger.info(F"Saving model to {output_model_file}" )
								torch.save(a     , a )
								logger.info(F"Model saved to {output_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Tuple				     =						(
						    F"{MODEL_NAME}_rank{accelerator.process_index}.bin"
						    if model_index == 0
						    else F"{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(a     , a )
						logger.info(F"Saving model to {output_model_file}" )
						torch.save(a     , a )
						logger.info(F"Model saved to {output_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:Dict				     =						os.path.join(a     , F"{MODEL_NAME}_{model_index}" )
						os.makedirs(a     , exist_ok=a )
						logger.info(F"Saving model to {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:int				     =						{"model": state_dict}

						dist_cp.save_state_dict(
						    state_dict=a     , storage_writer=dist_cp.FileSystemWriter(a )     , planner=DefaultSavePlanner()     , )
						logger.info(F"Model saved to {ckpt_dir}" )


def     lowercase	(    a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if type(a ) != FSDP and accelerator.process_index != 0:
								if not fsdp_plugin.sync_module_states:
										raise ValueError(
										    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
										    "initializing FSDP object" )
								return
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						F"{MODEL_NAME}.bin" if model_index == 0 else F"{MODEL_NAME}_{model_index}.bin"
						SCREAMING_SNAKE_CASE_				:int				     =						os.path.join(a     , a )
						logger.info(F"Loading model from {input_model_file}" )
						SCREAMING_SNAKE_CASE_				:Dict				     =						torch.load(a )
						logger.info(F"Model loaded from {input_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:int				     =						(
						    F"{MODEL_NAME}_rank{accelerator.process_index}.bin"
						    if model_index == 0
						    else F"{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:int				     =						os.path.join(a     , a )
						logger.info(F"Loading model from {input_model_file}" )
						SCREAMING_SNAKE_CASE_				:Dict				     =						torch.load(a )
						logger.info(F"Model loaded from {input_model_file}" )
				elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:List[str]				     =						(
						    os.path.join(a     , F"{MODEL_NAME}_{model_index}" )
						    if F"{MODEL_NAME}" not in input_dir
						    else input_dir
						)
						logger.info(F"Loading model from {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:Any				     =						{"model": model.state_dict()}
						dist_cp.load_state_dict(
						    state_dict=a     , storage_reader=dist_cp.FileSystemReader(a )     , planner=DefaultLoadPlanner()     , )
						SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						state_dict["model"]
						logger.info(F"Model loaded from {ckpt_dir}" )
				model.load_state_dict(a )


def     lowercase	(    a     , a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		os.makedirs(a     , exist_ok=a )
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				SCREAMING_SNAKE_CASE_				:str				     =						FSDP.optim_state_dict(a     , a )
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						if accelerator.process_index == 0:
								SCREAMING_SNAKE_CASE_				:List[str]				     =						(
								    F"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else F"{OPTIMIZER_NAME}_{optimizer_index}.bin"
								)
								SCREAMING_SNAKE_CASE_				:Optional[Any]				     =						os.path.join(a     , a )
								logger.info(F"Saving Optimizer state to {output_optimizer_file}" )
								torch.save(a     , a )
								logger.info(F"Optimizer state saved in {output_optimizer_file}" )
				else:
						SCREAMING_SNAKE_CASE_				:Any				     =						os.path.join(a     , F"{OPTIMIZER_NAME}_{optimizer_index}" )
						os.makedirs(a     , exist_ok=a )
						logger.info(F"Saving Optimizer state to {ckpt_dir}" )
						dist_cp.save_state_dict(
						    state_dict={"optimizer": optim_state}     , storage_writer=dist_cp.FileSystemWriter(a )     , planner=DefaultSavePlanner()     , )
						logger.info(F"Optimizer state saved in {ckpt_dir}" )


def     lowercase	(    a     , a     , a     , a     , a     , a=0 ):



		'''simple docstring'''




		accelerator.wait_for_everyone()
		with FSDP.state_dict_type(
		    a     , fsdp_plugin.state_dict_type     , fsdp_plugin.state_dict_config     , fsdp_plugin.optim_state_dict_config ):
				if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
						SCREAMING_SNAKE_CASE_				:List[str]				     =						None
						# below check should work but currently it isn't working (mostly opytorch issue),
						# in the meantime disabling it at the cost of excess memory usage
						# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
						SCREAMING_SNAKE_CASE_				:List[Any]				     =						(
						    F"{OPTIMIZER_NAME}.bin" if optimizer_index == 0 else F"{OPTIMIZER_NAME}_{optimizer_index}.bin"
						)
						SCREAMING_SNAKE_CASE_				:Dict				     =						os.path.join(a     , a )
						logger.info(F"Loading Optimizer state from {input_optimizer_file}" )
						SCREAMING_SNAKE_CASE_				:str				     =						torch.load(a )
						logger.info(F"Optimizer state loaded from {input_optimizer_file}" )
				else:
						SCREAMING_SNAKE_CASE_				:Any				     =						(
						    os.path.join(a     , F"{OPTIMIZER_NAME}_{optimizer_index}" )
						    if F"{OPTIMIZER_NAME}" not in input_dir
						    else input_dir
						)
						logger.info(F"Loading Optimizer from {ckpt_dir}" )
						SCREAMING_SNAKE_CASE_				:Union[str, Any]				     =						load_sharded_optimizer_state_dict(
						    model_state_dict=model.state_dict()     , optimizer_key="optimizer"     , storage_reader=dist_cp.FileSystemReader(a )     , )
						SCREAMING_SNAKE_CASE_				:List[str]				     =						optim_state["optimizer"]
						logger.info(F"Optimizer loaded from {ckpt_dir}" )
				SCREAMING_SNAKE_CASE_				:int				     =						FSDP.optim_state_dict_to_load(a     , a     , a )
				optimizer.load_state_dict(a )






