

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      BertJapaneseTokenizer
   lowerCamelCase__				=      False
   lowerCamelCase__				=      True





   def      _snake_case		(       self					:			Dict			):
       super().setUp()

       SCREAMING_SNAKE_CASE										=   [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "こんにちは",
           "こん",
           "にちは",
           "ばんは",
           "##こん",
           "##にちは",
           "##ばんは",
           "世界",
           "##世界",
           "、",
           "##、",
           "。",
           "##。",
       ]

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file     , "w"     , encoding="utf-8"			) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   "こんにちは 、 世界 。 こんばんは 、 世界 。"
       return input_text, output_text





   def      _snake_case		(       self					:			str     , __lowerCamelCase					:			Any			):
       SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE										=   self.get_input_output_texts(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode(__lowerCamelCase     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.decode(__lowerCamelCase     , clean_up_tokenization_spaces=__lowerCamelCase			)
       return text, ids





   def      _snake_case		(       self					:			Optional[int]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			List[Any]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			List[Any]			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file			)

       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="mecab"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			int			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="unidic_lite"			)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Optional[Any]			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(mecab_dic="unidic"			)
       except ModuleNotFoundError:
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(do_lower_case=__lowerCamelCase     , mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   def      _snake_case		(       self					:			Union[str, Any]			):
       try:
           SCREAMING_SNAKE_CASE										=   MecabTokenizer(
               do_lower_case=__lowerCamelCase     , normalize_text=__lowerCamelCase     , mecab_option="-d /usr/local/lib/mecab/dic/jumandic"			)
       except RuntimeError:
           # if dict doesn't exist in the system, previous code raises this error.
           return

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]     , )





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   MecabTokenizer(normalize_text=__lowerCamelCase     , mecab_dic="ipadic"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]     , )





   @require_sudachi
   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="sudachi"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   @require_sudachi
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="A"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国", "人", "参政", "権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="B"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国人", "参政権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(sudachi_dict_type="core"     , sudachi_split_mode="C"			)

       self.assertListEqual(tokenizer.tokenize("外国人参政権"			)     , ["外国人参政権"]			)





   @require_sudachi
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(do_lower_case=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(normalize_text=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]     , )





   @require_sudachi
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   SudachiTokenizer(trim_whitespace=__lowerCamelCase     , sudachi_dict_type="core"			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , word_tokenizer_type="jumanpp"			)
       self.assertIsNotNone(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。\nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize(__lowerCamelCase			)
       self.assertListEqual(__lowerCamelCase     , ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 12, 10, 14, 4, 9, 12, 10, 14]			)

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , "tokenizer.bin"			)
       with open(__lowerCamelCase     , "wb"			) as handle:
           pickle.dump(__lowerCamelCase     , __lowerCamelCase			)

       with open(__lowerCamelCase     , "rb"			) as handle:
           SCREAMING_SNAKE_CASE										=   pickle.load(__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer_new.tokenize(__lowerCamelCase			)

       self.assertListEqual(__lowerCamelCase     , __lowerCamelCase			)





   @require_jumanpp
   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(do_lower_case=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(normalize_text=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer(trim_whitespace=__lowerCamelCase			)

       self.assertListEqual(
           tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "			)     , ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]     , )





   @require_jumanpp
   def      _snake_case		(       self					:			List[Any]			):
       SCREAMING_SNAKE_CASE										=   JumanppTokenizer()

       self.assertListEqual(
           tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"			)     , ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]     , )





   def      _snake_case		(       self					:			int			):
       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

       SCREAMING_SNAKE_CASE										=   {}
       for i, token in enumerate(__lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   i
       SCREAMING_SNAKE_CASE										=   WordpieceTokenizer(vocab=__lowerCamelCase     , unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)     , []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)     , ["こんにちは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは"			)     , ["こん", "##ばんは"]			)

       self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"			)     , ["こん", "##ばんは", "[UNK]", "こんにちは"]			)





   def      _snake_case		(       self					:			Any			):
       SCREAMING_SNAKE_CASE										=   BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"			)
       SCREAMING_SNAKE_CASE										=   tokenizer.subword_tokenizer

       SCREAMING_SNAKE_CASE										=   subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"			)
       self.assertListEqual(__lowerCamelCase     , ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]			)

       SCREAMING_SNAKE_CASE										=   subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"			)
       self.assertListEqual(__lowerCamelCase     , ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]			)





   def      _snake_case		(       self					:			Optional[Any]			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.encode("ありがとう。"     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode("どういたしまして。"     , add_special_tokens=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase     , __lowerCamelCase			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   __snake_case					,       unittest.TestCase		):







   '''simple docstring'''




   lowerCamelCase__				=      BertJapaneseTokenizer
   lowerCamelCase__				=      False





   def      _snake_case		(       self					:			Optional[Any]			):
       super().setUp()

       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE										=   os.path.join(self.tmpdirname     , VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file     , "w"     , encoding="utf-8"			) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)





   def      _snake_case		(       self					:			str     , **__lowerCamelCase					:			Optional[int]			):
       return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     , subword_tokenizer_type="character"     , **__lowerCamelCase			)





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   "こんにちは、世界。 \nこんばんは、世界。"
       SCREAMING_SNAKE_CASE										=   "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
       return input_text, output_text





   def      _snake_case		(       self					:			Tuple			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Tuple			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Any			):
       pass  # TODO add if relevant





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class(self.vocab_file     , subword_tokenizer_type="character"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"			)
       self.assertListEqual(
           __lowerCamelCase     , ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]			)
       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__lowerCamelCase			)     , [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]			)





   def      _snake_case		(       self					:			Optional[int]			):
       SCREAMING_SNAKE_CASE										=   ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

       SCREAMING_SNAKE_CASE										=   {}
       for i, token in enumerate(__lowerCamelCase			):
           SCREAMING_SNAKE_CASE										=   i
       SCREAMING_SNAKE_CASE										=   CharacterTokenizer(vocab=__lowerCamelCase     , unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)     , []			)

       self.assertListEqual(tokenizer.tokenize("こんにちは"			)     , ["こ", "ん", "に", "ち", "は"]			)

       self.assertListEqual(tokenizer.tokenize("こんにちほ"			)     , ["こ", "ん", "に", "ち", "[UNK]"]			)





   def      _snake_case		(       self					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"			)

       SCREAMING_SNAKE_CASE										=   tokenizer.encode("ありがとう。"     , add_special_tokens=__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.encode("どういたしまして。"     , add_special_tokens=__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   tokenizer.build_inputs_with_special_tokens(__lowerCamelCase     , __lowerCamelCase			)

       # 2 is for "[CLS]", 3 is for "[SEP]"
       assert encoded_sentence == [2] + text + [3]
       assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   def      _snake_case		(       self					:			str			):
       SCREAMING_SNAKE_CASE										=   "cl-tohoku/bert-base-japanese"
       SCREAMING_SNAKE_CASE										=   AutoTokenizer.from_pretrained(__lowerCamelCase			)
       self.assertIsInstance(__lowerCamelCase     , __lowerCamelCase			)





class 			_SCREAMING_SNAKE_CASE (   unittest.TestCase		):







   '''simple docstring'''




   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   "cl-tohoku/bert-base-japanese"
       with self.assertLogs("transformers"     , level="WARNING"			) as cm:
           BertTokenizer.from_pretrained(__lowerCamelCase			)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."			)			)
       SCREAMING_SNAKE_CASE										=   "bert-base-cased"
       with self.assertLogs("transformers"     , level="WARNING"			) as cm:
           BertJapaneseTokenizer.from_pretrained(__lowerCamelCase			)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."			)			)







from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


__a   :							Union[str, Any]						= logging.get_logger(__name__)


# General docstring
__a   :							List[str]						= "MobileNetV1Config"

# Base docstring
__a   :							int						= "google/mobilenet_v1_1.0_224"
__a   :							List[Any]						= [1, 1024, 7, 7]

# Image classification docstring
__a   :							Optional[int]						= "google/mobilenet_v1_1.0_224"
__a   :							List[Any]						= "tabby, tabby cat"


__a   :							Union[str, Any]						= [
    "google/mobilenet_v1_1.0_224",
    "google/mobilenet_v1_0.75_192",
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      List[str]      ,					__lowercase :      str      ,					__lowercase :      Union[str, Any]=None     )    ->   Any:





    """simple docstring"""

    __A       =						{}

    if isinstance(__lowercase      ,					__lowercase     ):
        __A       =						model.mobilenet_va
    else:
        __A       =						model

    __A       =						"""MobilenetV1/Conv2d_0/"""
    __A       =						backbone.conv_stem.convolution.weight
    __A       =						backbone.conv_stem.normalization.bias
    __A       =						backbone.conv_stem.normalization.weight
    __A       =						backbone.conv_stem.normalization.running_mean
    __A       =						backbone.conv_stem.normalization.running_var

    for i in range(1_3     ):
        __A       =						i + 1
        __A       =						i * 2

        __A       =						backbone.layer[pt_index]
        __A       =						f"MobilenetV1/Conv2d_{tf_index}_depthwise/"
        __A       =						pointer.convolution.weight
        __A       =						pointer.normalization.bias
        __A       =						pointer.normalization.weight
        __A       =						pointer.normalization.running_mean
        __A       =						pointer.normalization.running_var

        __A       =						backbone.layer[pt_index + 1]
        __A       =						f"MobilenetV1/Conv2d_{tf_index}_pointwise/"
        __A       =						pointer.convolution.weight
        __A       =						pointer.normalization.bias
        __A       =						pointer.normalization.weight
        __A       =						pointer.normalization.running_mean
        __A       =						pointer.normalization.running_var

    if isinstance(__lowercase      ,					__lowercase     ):
        __A       =						"""MobilenetV1/Logits/Conv2d_1c_1x1/"""
        __A       =						model.classifier.weight
        __A       =						model.classifier.bias

    return tf_to_pt_map



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      Optional[Any]      ,					__lowercase :      int      ,					__lowercase :      int     )    ->   str:





    """simple docstring"""

    try:
        import numpy as np
        import tensorflow as tf
    except ImportError:
        logger.error(
            """Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see """
            """https://www.tensorflow.org/install/ for installation instructions."""     )
        raise

    # Load weights from TF model
    __A       =						tf.train.list_variables(__lowercase     )
    __A       =						{}
    for name, shape in init_vars:
        logger.info(f"Loading TF weight {name} with shape {shape}"     )
        __A       =						tf.train.load_variable(__lowercase      ,					__lowercase     )
        __A       =						array

    # Build TF to PyTorch weights loading map
    __A       =						_build_tf_to_pytorch_map(__lowercase      ,					__lowercase      ,					__lowercase     )

    for name, pointer in tf_to_pt_map.items():
        logger.info(f"Importing {name}"     )
        if name not in tf_weights:
            logger.info(f"{name} not in tf pre-trained weights, skipping"     )
            continue

        __A       =						tf_weights[name]

        if "depthwise_weights" in name:
            logger.info("""Transposing depthwise"""     )
            __A       =						np.transpose(__lowercase      ,					(2, 3, 0, 1)     )
        elif "weights" in name:
            logger.info("""Transposing"""     )
            if len(pointer.shape     ) == 2:  # copying into linear layer
                __A       =						array.squeeze().transpose()
            else:
                __A       =						np.transpose(__lowercase      ,					(3, 2, 0, 1)     )

        if pointer.shape != array.shape:
            raise ValueError(f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched"     )

        logger.info(f"Initialize PyTorch weight {name} {array.shape}"     )
        __A       =						torch.from_numpy(__lowercase     )

        tf_weights.pop(__lowercase      ,					__lowercase     )
        tf_weights.pop(name + """/RMSProp"""      ,					__lowercase     )
        tf_weights.pop(name + """/RMSProp_1"""      ,					__lowercase     )
        tf_weights.pop(name + """/ExponentialMovingAverage"""      ,					__lowercase     )

    logger.info(f"Weights not copied to PyTorch model: {', '.join(tf_weights.keys()     )}"     )
    return model



def       _SCREAMING_SNAKE_CASE    (			__lowercase :      torch.Tensor      ,					__lowercase :      nn.Convad     )    ->   torch.Tensor:





    """simple docstring"""

    __A     ,						__A       =						features.shape[-2:]
    __A     ,						__A       =						conv_layer.stride
    __A     ,						__A       =						conv_layer.kernel_size

    if in_height % stride_height == 0:
        __A       =						max(kernel_height - stride_height      ,					0     )
    else:
        __A       =						max(kernel_height - (in_height % stride_height)      ,					0     )

    if in_width % stride_width == 0:
        __A       =						max(kernel_width - stride_width      ,					0     )
    else:
        __A       =						max(kernel_width - (in_width % stride_width)      ,					0     )

    __A       =						pad_along_width // 2
    __A       =						pad_along_width - pad_left
    __A       =						pad_along_height // 2
    __A       =						pad_along_height - pad_top

    __A       =						(pad_left, pad_right, pad_top, pad_bottom)
    return nn.functional.pad(__lowercase      ,					__lowercase      ,					"""constant"""      ,					0.0     )


class    __lowercase   (						nn.Module	):







     '''simple docstring'''




     def __init__(					self							:      int			,						UpperCamelCase_							:      MobileNetVaConfig			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      Optional[int] = 1			,						UpperCamelCase_							:      Optional[int] = 1			,						UpperCamelCase_							:      bool = False			,						UpperCamelCase_							:      Optional[bool] = True			,						UpperCamelCase_							:      Optional[bool or str] = True			,						):


         """simple docstring"""



         super().__init__()
         __A       =						config

         if in_channels % groups != 0:
             raise ValueError(F"Input channels ({in_channels}) are not divisible by {groups} groups."					)
         if out_channels % groups != 0:
             raise ValueError(F"Output channels ({out_channels}) are not divisible by {groups} groups."					)

         __A       =						0 if config.tf_padding else int((kernel_size - 1) / 2					)

         __A       =						nn.Convad(
             in_channels=UpperCamelCase_			,						out_channels=UpperCamelCase_			,						kernel_size=UpperCamelCase_			,						stride=UpperCamelCase_			,						padding=UpperCamelCase_			,						groups=UpperCamelCase_			,						bias=UpperCamelCase_			,						padding_mode="""zeros"""			,						)

         if use_normalization:
             __A       =						nn.BatchNormad(
                 num_features=UpperCamelCase_			,						eps=config.layer_norm_eps			,						momentum=0.9997			,						affine=UpperCamelCase_			,						track_running_stats=UpperCamelCase_			,						)
         else:
             __A       =						None

         if use_activation:
             if isinstance(UpperCamelCase_			,						UpperCamelCase_					):
                 __A       =						ACTaFN[use_activation]
             elif isinstance(config.hidden_act			,						UpperCamelCase_					):
                 __A       =						ACTaFN[config.hidden_act]
             else:
                 __A       =						config.hidden_act
         else:
             __A       =						None





     def 		lowerCAmelCase_  (					self							:      Union[str, Any]			,						UpperCamelCase_							:      torch.Tensor					):


         """simple docstring"""



         if self.config.tf_padding:
             __A       =						apply_tf_padding(UpperCamelCase_			,						self.convolution					)
         __A       =						self.convolution(UpperCamelCase_					)
         if self.normalization is not None:
             __A       =						self.normalization(UpperCamelCase_					)
         if self.activation is not None:
             __A       =						self.activation(UpperCamelCase_					)
         return features


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					MobileNetVaConfig
     SCREAMING_SNAKE_CASE     =					load_tf_weights_in_mobilenet_va
     SCREAMING_SNAKE_CASE     =					"mobilenet_v1"
     SCREAMING_SNAKE_CASE     =					"pixel_values"
     SCREAMING_SNAKE_CASE     =					False




     def 		lowerCAmelCase_  (					self							:      Optional[Any]			,						UpperCamelCase_							:      Union[nn.Linear, nn.Convad]					):


         """simple docstring"""



         if isinstance(UpperCamelCase_			,						(nn.Linear, nn.Convad)					):
             module.weight.data.normal_(mean=0.0			,						std=self.config.initializer_range					)
             if module.bias is not None:
                 module.bias.data.zero_()
         elif isinstance(UpperCamelCase_			,						nn.BatchNormad					):
             module.bias.data.zero_()
             module.weight.data.fill_(1.0					)
__a   :							Tuple						= R"\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

__a   :							int						= R"\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`MobileNetV1ImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n"


@add_start_docstrings(
    "The bare MobileNetV1 model outputting raw hidden-states without any specific head on top."      ,   lowercase_      ,   )
class    __lowercase   (						lowercase_	):







     '''simple docstring'''




     def __init__(					self							:      int			,						UpperCamelCase_							:      MobileNetVaConfig			,						UpperCamelCase_							:      bool = True					):


         """simple docstring"""



         super().__init__(UpperCamelCase_					)
         __A       =						config

         __A       =						32
         __A       =						max(int(depth * config.depth_multiplier					)			,						config.min_depth					)

         __A       =						MobileNetVaConvLayer(
             UpperCamelCase_			,						in_channels=config.num_channels			,						out_channels=UpperCamelCase_			,						kernel_size=3			,						stride=2			,						)

         __A       =						[1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

         __A       =						nn.ModuleList()
         for i in range(13					):
             __A       =						out_channels

             if strides[i] == 2 or i == 0:
                 depth *= 2
                 __A       =						max(int(depth * config.depth_multiplier					)			,						config.min_depth					)

             self.layer.append(
                 MobileNetVaConvLayer(
                     UpperCamelCase_			,						in_channels=UpperCamelCase_			,						out_channels=UpperCamelCase_			,						kernel_size=3			,						stride=strides[i]			,						groups=UpperCamelCase_			,						)					)

             self.layer.append(
                 MobileNetVaConvLayer(
                     UpperCamelCase_			,						in_channels=UpperCamelCase_			,						out_channels=UpperCamelCase_			,						kernel_size=1			,						)					)

         __A       =						nn.AdaptiveAvgPoolad((1, 1)					) if add_pooling_layer else None

         # Initialize weights and apply final processing
         self.post_init()




     def 		lowerCAmelCase_  (					self							:      Optional[int]			,						UpperCamelCase_							:      Optional[int]					):


         """simple docstring"""



         raise NotImplementedError





     @add_start_docstrings_to_model_forward(UpperCamelCase_					)
     @add_code_sample_docstrings(
         checkpoint=_CHECKPOINT_FOR_DOC			,						output_type=UpperCamelCase_			,						config_class=_CONFIG_FOR_DOC			,						modality="""vision"""			,						expected_output=_EXPECTED_OUTPUT_SHAPE			,						)
     def 		lowerCAmelCase_  (					self							:      Any			,						UpperCamelCase_							:      Optional[torch.Tensor] = None			,						UpperCamelCase_							:      Optional[bool] = None			,						UpperCamelCase_							:      Optional[bool] = None			,						):


         """simple docstring"""



         __A       =						(
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
         )
         __A       =						return_dict if return_dict is not None else self.config.use_return_dict

         if pixel_values is None:
             raise ValueError("""You have to specify pixel_values"""					)

         __A       =						self.conv_stem(UpperCamelCase_					)

         __A       =						() if output_hidden_states else None

         for i, layer_module in enumerate(self.layer					):
             __A       =						layer_module(UpperCamelCase_					)

             if output_hidden_states:
                 __A       =						all_hidden_states + (hidden_states,)

         __A       =						hidden_states

         if self.pooler is not None:
             __A       =						torch.flatten(self.pooler(UpperCamelCase_					)			,						start_dim=1					)
         else:
             __A       =						None

         if not return_dict:
             return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None					)

         return BaseModelOutputWithPoolingAndNoAttention(
             last_hidden_state=UpperCamelCase_			,						pooler_output=UpperCamelCase_			,						hidden_states=UpperCamelCase_			,						)


@add_start_docstrings(
    "\n    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "      ,   lowercase_      ,   )
class    __lowercase   (						lowercase_	):







     '''simple docstring'''




     def __init__(					self							:      Optional[int]			,						UpperCamelCase_							:      MobileNetVaConfig					):


         """simple docstring"""



         super().__init__(UpperCamelCase_					)

         __A       =						config.num_labels
         __A       =						MobileNetVaModel(UpperCamelCase_					)

         __A       =						self.mobilenet_va.layer[-1].convolution.out_channels

         # Classifier head
         __A       =						nn.Dropout(config.classifier_dropout_prob			,						inplace=UpperCamelCase_					)
         __A       =						nn.Linear(UpperCamelCase_			,						config.num_labels					) if config.num_labels > 0 else nn.Identity()

         # Initialize weights and apply final processing
         self.post_init()





     @add_start_docstrings_to_model_forward(UpperCamelCase_					)
     @add_code_sample_docstrings(
         checkpoint=_IMAGE_CLASS_CHECKPOINT			,						output_type=UpperCamelCase_			,						config_class=_CONFIG_FOR_DOC			,						expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT			,						)
     def 		lowerCAmelCase_  (					self							:      Tuple			,						UpperCamelCase_							:      Optional[torch.Tensor] = None			,						UpperCamelCase_							:      Optional[bool] = None			,						UpperCamelCase_							:      Optional[torch.Tensor] = None			,						UpperCamelCase_							:      Optional[bool] = None			,						):


         """simple docstring"""



         __A       =						return_dict if return_dict is not None else self.config.use_return_dict

         __A       =						self.mobilenet_va(UpperCamelCase_			,						output_hidden_states=UpperCamelCase_			,						return_dict=UpperCamelCase_					)

         __A       =						outputs.pooler_output if return_dict else outputs[1]

         __A       =						self.classifier(self.dropout(UpperCamelCase_					)					)

         __A       =						None
         if labels is not None:
             if self.config.problem_type is None:
                 if self.num_labels == 1:
                     __A       =						"""regression"""
                 elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                     __A       =						"""single_label_classification"""
                 else:
                     __A       =						"""multi_label_classification"""

             if self.config.problem_type == "regression":
                 __A       =						MSELoss()
                 if self.num_labels == 1:
                     __A       =						loss_fct(logits.squeeze()			,						labels.squeeze()					)
                 else:
                     __A       =						loss_fct(UpperCamelCase_			,						UpperCamelCase_					)
             elif self.config.problem_type == "single_label_classification":
                 __A       =						CrossEntropyLoss()
                 __A       =						loss_fct(logits.view(-1			,						self.num_labels					)			,						labels.view(-1					)					)
             elif self.config.problem_type == "multi_label_classification":
                 __A       =						BCEWithLogitsLoss()
                 __A       =						loss_fct(UpperCamelCase_			,						UpperCamelCase_					)

         if not return_dict:
             __A       =						(logits,) + outputs[2:]
             return ((loss,) + output) if loss is not None else output

         return ImageClassifierOutputWithNoAttention(
             loss=UpperCamelCase_			,						logits=UpperCamelCase_			,						hidden_states=outputs.hidden_states			,						)




