


'''simple docstring'''




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 			_lowercase (	__lowercase						,					unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  Union[str, Any]          =	BertJapaneseTokenizer
   _SCREAMING_SNAKE_CASE  :  str          =	False
   _SCREAMING_SNAKE_CASE  :  Any          =	True
   def a  (						self       :      Tuple      ) ->							List[Any]:
        super().setUp()

        __snake_case           =						[
            '[UNK]',
            '[CLS]',
            '[SEP]',
            'こんにちは',
            'こん',
            'にちは',
            'ばんは',
            '##こん',
            '##にちは',
            '##ばんは',
            '世界',
            '##世界',
            '、',
            '##、',
            '。',
            '##。',
        ]

        __snake_case           =						os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['vocab_file']      )
        with open(self.vocab_file       ,   'w'       ,   encoding='utf-8'      ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							Optional[int]:
        __snake_case           =						'こんにちは、世界。 \nこんばんは、世界。'
        __snake_case           =						'こんにちは 、 世界 。 こんばんは 、 世界 。'
        return input_text, output_text
   def a  (						self       :      Union[str, Any]       ,   SCREAMING_SNAKE_CASE_       :      List[str]      ) ->							Union[str, Any]:
        __snake_case      ,       __snake_case           =						self.get_input_output_texts(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.encode(SCREAMING_SNAKE_CASE_       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.decode(SCREAMING_SNAKE_CASE_       ,   clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_      )
        return text, ids
   def a  (						self       :      int      ) ->							Union[str, Any]:
        pass  # TODO add if relevant
   def a  (						self       :      Dict      ) ->							Union[str, Any]:
        pass  # TODO add if relevant
   def a  (						self       :      str      ) ->							int:
        pass  # TODO add if relevant
   def a  (						self       :      List[Any]      ) ->							List[Any]:
        __snake_case           =						self.tokenizer_class(self.vocab_file      )

        __snake_case           =						tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]      )
   def a  (						self       :      List[str]      ) ->							Optional[int]:
        __snake_case           =						self.tokenizer_class(self.vocab_file       ,   word_tokenizer_type='mecab'      )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						'こんにちは、世界。\nこんばんは、世界。'
        __snake_case           =						tokenizer.tokenize(SCREAMING_SNAKE_CASE_      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

        __snake_case           =						os.path.join(self.tmpdirname       ,   'tokenizer.bin'      )
        with open(SCREAMING_SNAKE_CASE_       ,   'wb'      ) as handle:
             pickle.dump(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        with open(SCREAMING_SNAKE_CASE_       ,   'rb'      ) as handle:
             __snake_case           =						pickle.load(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer_new.tokenize(SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Any      ) ->							Union[str, Any]:
        __snake_case           =						MecabTokenizer(mecab_dic='ipadic'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,   )
   def a  (						self       :      Optional[Any]      ) ->							Tuple:
        try:
             __snake_case           =						MecabTokenizer(mecab_dic='unidic_lite'      )
        except ModuleNotFoundError:
             return

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,   )
   def a  (						self       :      int      ) ->							Optional[Any]:
        try:
             __snake_case           =						MecabTokenizer(mecab_dic='unidic'      )
        except ModuleNotFoundError:
             return

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,   )
   def a  (						self       :      List[Any]      ) ->							Optional[Any]:
        __snake_case           =						MecabTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,   mecab_dic='ipadic'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,   )
   def a  (						self       :      Union[str, Any]      ) ->							int:
        try:
             __snake_case           =						MecabTokenizer(
                 do_lower_case=SCREAMING_SNAKE_CASE_       ,   normalize_text=SCREAMING_SNAKE_CASE_       ,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'      )
        except RuntimeError:
             # if dict doesn't exist in the system, previous code raises this error.
             return

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']       ,   )
   def a  (						self       :      List[str]      ) ->							Union[str, Any]:
        __snake_case           =						MecabTokenizer(normalize_text=SCREAMING_SNAKE_CASE_       ,   mecab_dic='ipadic'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']       ,   )
   @require_sudachi
   def a  (						self       :      Tuple      ) ->							List[Any]:
        __snake_case           =						self.tokenizer_class(self.vocab_file       ,   word_tokenizer_type='sudachi'      )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						'こんにちは、世界。\nこんばんは、世界。'
        __snake_case           =						tokenizer.tokenize(SCREAMING_SNAKE_CASE_      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

        __snake_case           =						os.path.join(self.tmpdirname       ,   'tokenizer.bin'      )
        with open(SCREAMING_SNAKE_CASE_       ,   'wb'      ) as handle:
             pickle.dump(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        with open(SCREAMING_SNAKE_CASE_       ,   'rb'      ) as handle:
             __snake_case           =						pickle.load(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer_new.tokenize(SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
   @require_sudachi
   def a  (						self       :      Tuple      ) ->							Optional[int]:
        __snake_case           =						SudachiTokenizer(sudachi_dict_type='core'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']       ,   )
   @require_sudachi
   def a  (						self       :      List[Any]      ) ->							List[str]:
        __snake_case           =						SudachiTokenizer(sudachi_dict_type='core'       ,   sudachi_split_mode='A'      )

        self.assertListEqual(tokenizer.tokenize('外国人参政権'      )       ,   ['外国', '人', '参政', '権']      )
   @require_sudachi
   def a  (						self       :      Optional[Any]      ) ->							Any:
        __snake_case           =						SudachiTokenizer(sudachi_dict_type='core'       ,   sudachi_split_mode='B'      )

        self.assertListEqual(tokenizer.tokenize('外国人参政権'      )       ,   ['外国人', '参政権']      )
   @require_sudachi
   def a  (						self       :      Dict      ) ->							Optional[Any]:
        __snake_case           =						SudachiTokenizer(sudachi_dict_type='core'       ,   sudachi_split_mode='C'      )

        self.assertListEqual(tokenizer.tokenize('外国人参政権'      )       ,   ['外国人参政権']      )
   @require_sudachi
   def a  (						self       :      int      ) ->							Tuple:
        __snake_case           =						SudachiTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,   sudachi_dict_type='core'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']       ,   )
   @require_sudachi
   def a  (						self       :      str      ) ->							Tuple:
        __snake_case           =						SudachiTokenizer(normalize_text=SCREAMING_SNAKE_CASE_       ,   sudachi_dict_type='core'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']       ,   )
   @require_sudachi
   def a  (						self       :      str      ) ->							Union[str, Any]:
        __snake_case           =						SudachiTokenizer(trim_whitespace=SCREAMING_SNAKE_CASE_       ,   sudachi_dict_type='core'      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']       ,   )
   @require_jumanpp
   def a  (						self       :      Union[str, Any]      ) ->							List[str]:
        __snake_case           =						self.tokenizer_class(self.vocab_file       ,   word_tokenizer_type='jumanpp'      )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						'こんにちは、世界。\nこんばんは、世界。'
        __snake_case           =						tokenizer.tokenize(SCREAMING_SNAKE_CASE_      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']      )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [3, 12, 10, 14, 4, 9, 12, 10, 14]      )

        __snake_case           =						os.path.join(self.tmpdirname       ,   'tokenizer.bin'      )
        with open(SCREAMING_SNAKE_CASE_       ,   'wb'      ) as handle:
             pickle.dump(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        with open(SCREAMING_SNAKE_CASE_       ,   'rb'      ) as handle:
             __snake_case           =						pickle.load(SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer_new.tokenize(SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
   @require_jumanpp
   def a  (						self       :      Union[str, Any]      ) ->							Dict:
        __snake_case           =						JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,   )
   @require_jumanpp
   def a  (						self       :      Any      ) ->							Optional[int]:
        __snake_case           =						JumanppTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,   )
   @require_jumanpp
   def a  (						self       :      List[Any]      ) ->							Any:
        __snake_case           =						JumanppTokenizer(normalize_text=SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']       ,   )
   @require_jumanpp
   def a  (						self       :      Any      ) ->							int:
        __snake_case           =						JumanppTokenizer(trim_whitespace=SCREAMING_SNAKE_CASE_      )

        self.assertListEqual(
            tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '      )       ,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']       ,   )
   @require_jumanpp
   def a  (						self       :      Dict      ) ->							Dict:
        __snake_case           =						JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'      )       ,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']       ,   )
   def a  (						self       :      Optional[Any]      ) ->							int:
        __snake_case           =						['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

        __snake_case           =						{}
        for i, token in enumerate(SCREAMING_SNAKE_CASE_      ):
             __snake_case           =						i
        __snake_case           =						WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_       ,   unk_token='[UNK]'      )

        self.assertListEqual(tokenizer.tokenize(''      )       ,   []      )

        self.assertListEqual(tokenizer.tokenize('こんにちは'      )       ,   ['こんにちは']      )

        self.assertListEqual(tokenizer.tokenize('こんばんは'      )       ,   ['こん', '##ばんは']      )

        self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'      )       ,   ['こん', '##ばんは', '[UNK]', 'こんにちは']      )
   def a  (						self       :      Dict      ) ->							Union[str, Any]:
        __snake_case           =						BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'      )
        __snake_case           =						tokenizer.subword_tokenizer

        __snake_case           =						subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']      )

        __snake_case           =						subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'      )
        self.assertListEqual(SCREAMING_SNAKE_CASE_       ,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']      )





   def a  (						self       :      Any      ) ->							Any:
        __snake_case           =						self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'      )

        __snake_case           =						tokenizer.encode('ありがとう。'       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.encode('どういたしまして。'       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			_lowercase (	__lowercase						,					unittest.TestCase							):
   _SCREAMING_SNAKE_CASE  :  Tuple          =	BertJapaneseTokenizer
   _SCREAMING_SNAKE_CASE  :  List[Any]          =	False
   def a  (						self       :      Any      ) ->							int:
        super().setUp()

        __snake_case           =						['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

        __snake_case           =						os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['vocab_file']      )
        with open(self.vocab_file       ,   'w'       ,   encoding='utf-8'      ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )
   def a  (						self       :      List[Any]       ,   **SCREAMING_SNAKE_CASE_       :      Optional[int]      ) ->							List[str]:
        return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,   subword_tokenizer_type='character'       ,   **SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Any       ,   SCREAMING_SNAKE_CASE_       :      Dict      ) ->							Tuple:
        __snake_case           =						'こんにちは、世界。 \nこんばんは、世界。'
        __snake_case           =						'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
        return input_text, output_text
   def a  (						self       :      Optional[Any]      ) ->							List[Any]:
        pass  # TODO add if relevant
   def a  (						self       :      int      ) ->							Any:
        pass  # TODO add if relevant
   def a  (						self       :      Dict      ) ->							List[str]:
        pass  # TODO add if relevant
   def a  (						self       :      Optional[int]      ) ->							int:
        __snake_case           =						self.tokenizer_class(self.vocab_file       ,   subword_tokenizer_type='character'      )

        __snake_case           =						tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'      )
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_       ,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']      )
        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )       ,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]      )
   def a  (						self       :      Union[str, Any]      ) ->							List[str]:
        __snake_case           =						['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

        __snake_case           =						{}
        for i, token in enumerate(SCREAMING_SNAKE_CASE_      ):
             __snake_case           =						i
        __snake_case           =						CharacterTokenizer(vocab=SCREAMING_SNAKE_CASE_       ,   unk_token='[UNK]'      )

        self.assertListEqual(tokenizer.tokenize(''      )       ,   []      )

        self.assertListEqual(tokenizer.tokenize('こんにちは'      )       ,   ['こ', 'ん', 'に', 'ち', 'は']      )

        self.assertListEqual(tokenizer.tokenize('こんにちほ'      )       ,   ['こ', 'ん', 'に', 'ち', '[UNK]']      )





   def a  (						self       :      Any      ) ->							Any:
        __snake_case           =						self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'      )

        __snake_case           =						tokenizer.encode('ありがとう。'       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.encode('どういたしまして。'       ,   add_special_tokens=SCREAMING_SNAKE_CASE_      )

        __snake_case           =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_      )
        __snake_case           =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			_lowercase (	unittest.TestCase							):
   def a  (						self       :      Dict      ) ->							int:
        __snake_case           =						'cl-tohoku/bert-base-japanese'
        __snake_case           =						AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )




class 			_lowercase (	unittest.TestCase							):
   def a  (						self       :      Optional[int]      ) ->							int:
        __snake_case           =						'cl-tohoku/bert-base-japanese'
        with self.assertLogs('transformers'       ,   level='WARNING'      ) as cm:
             BertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
             self.assertTrue(
                 cm.records[0].message.startswith(
                     'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                     ' is called from.'      )      )
        __snake_case           =						'bert-base-cased'
        with self.assertLogs('transformers'       ,   level='WARNING'      ) as cm:
             BertJapaneseTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
             self.assertTrue(
                 cm.records[0].message.startswith(
                     'The tokenizer class you load from this checkpoint is not the same type as the class this function'
                     ' is called from.'      )      )






"""simple docstring"""




import argparse
import re

import requests
import torch

# git clone https://github.com/salesforce/BLIP.git
from models.blip import blip_decoder
from models.blip_itm import blip_itm
from models.blip_vqa import blip_vqa
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode

from transformers import (
    BertTokenizer,
    BlipConfig,
    BlipForConditionalGeneration,
    BlipForImageTextRetrieval,
    BlipForQuestionAnswering,
)



def 							_lowerCamelCase			(			__a,				__a     ):
		SCREAMING_SNAKE_CASE_    			=			'''https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg'''
		SCREAMING_SNAKE_CASE_    			=			Image.open(requests.get(__a,				stream=__a     ).raw     ).convert('''RGB'''     )

		SCREAMING_SNAKE_CASE_    			=			transforms.Compose(
		    [
		        transforms.Resize((image_size, image_size),				interpolation=InterpolationMode.BICUBIC     ),
		        transforms.ToTensor(),
		        transforms.Normalize((0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3),				(0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1)     ),
		    ]     )
		SCREAMING_SNAKE_CASE_    			=			transform(__a     ).unsqueeze(0     ).to(__a     )
		return image



def 							_lowerCamelCase			(			__a     ):
		if "visual_encoder" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub('''visual_encoder*''',				'''vision_model.encoder''',				__a     )
		if "blocks" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''blocks''',				'''layers''',				__a     )
		if "attn" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''attn''',				'''self_attn''',				__a     )
		if "norm1" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''norm1''',				'''layer_norm1''',				__a     )
		if "norm2" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''norm2''',				'''layer_norm2''',				__a     )
		if "encoder.norm" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''encoder.norm''',				'''post_layernorm''',				__a     )
		if "encoder.patch_embed.proj" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''encoder.patch_embed.proj''',				'''embeddings.patch_embedding''',				__a     )

		if "encoder.pos_embed" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''encoder.pos_embed''',				'''embeddings.position_embedding''',				__a     )
		if "encoder.cls_token" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''encoder.cls_token''',				'''embeddings.class_embedding''',				__a     )

		if "self_attn" in key:
				SCREAMING_SNAKE_CASE_    			=			re.sub(r'''self_attn.proj''',				'''self_attn.projection''',				__a     )

		return key



@torch.no_grad()
def 							_lowerCamelCase			(			__a,				__a=None     ):
		if config_path is not None:
				SCREAMING_SNAKE_CASE_    			=			BlipConfig.from_pretrained(__a     )
		else:
				SCREAMING_SNAKE_CASE_    			=			BlipConfig(projection_dim=512,				text_config={},				vision_config={}     )

		SCREAMING_SNAKE_CASE_    			=			BlipForConditionalGeneration(__a     ).eval()

		SCREAMING_SNAKE_CASE_    			=			'''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth'''

		SCREAMING_SNAKE_CASE_    			=			blip_decoder(pretrained=__a,				image_size=384,				vit='''base'''     )
		SCREAMING_SNAKE_CASE_    			=			pt_model.eval()

		SCREAMING_SNAKE_CASE_    			=			pt_model.state_dict()
		for key in modified_state_dict.copy():
				SCREAMING_SNAKE_CASE_    			=			modified_state_dict.pop(__a     )
				SCREAMING_SNAKE_CASE_    			=			rename_key(__a     )
				SCREAMING_SNAKE_CASE_    			=			value

		hf_model.load_state_dict(__a     )

		SCREAMING_SNAKE_CASE_    			=			384
		SCREAMING_SNAKE_CASE_    			=			load_demo_image(image_size=__a,				device='''cpu'''     )
		SCREAMING_SNAKE_CASE_    			=			BertTokenizer.from_pretrained('''bert-base-uncased'''     )
		SCREAMING_SNAKE_CASE_    			=			tokenizer(['''a picture of''']     ).input_ids

		SCREAMING_SNAKE_CASE_    			=			hf_model.generate(__a,				__a     )

		assert out[0].tolist() == [30_522, 1_037, 3_861, 1_997, 1_037, 2_450, 3_564, 2_006, 1_996, 3_509, 2_007, 2_014, 3_899, 102]

		SCREAMING_SNAKE_CASE_    			=			hf_model.generate(__a     )

		assert out[0].tolist() == [30_522, 1_037, 2_450, 3_564, 2_006, 1_996, 3_509, 2_007, 2_014, 3_899, 102]

		if pytorch_dump_folder_path is not None:
				hf_model.save_pretrained(__a     )

		# model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_vqa.pth'
		SCREAMING_SNAKE_CASE_    			=			(
		    '''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth'''
		)

		SCREAMING_SNAKE_CASE_    			=			blip_vqa(pretrained=__a,				image_size=__a,				vit='''base'''     )
		vqa_model.eval()

		SCREAMING_SNAKE_CASE_    			=			vqa_model.state_dict()
		for key in modified_state_dict.copy():
				SCREAMING_SNAKE_CASE_    			=			modified_state_dict.pop(__a     )
				SCREAMING_SNAKE_CASE_    			=			rename_key(__a     )
				SCREAMING_SNAKE_CASE_    			=			value

		SCREAMING_SNAKE_CASE_    			=			BlipForQuestionAnswering(__a     )

		hf_vqa_model.load_state_dict(__a     )

		SCREAMING_SNAKE_CASE_    			=			['''How many dogs are in this image?''']
		SCREAMING_SNAKE_CASE_    			=			tokenizer(__a,				return_tensors='''pt'''     ).input_ids

		SCREAMING_SNAKE_CASE_    			=			hf_vqa_model.generate(__a,				__a     )
		print(tokenizer.decode(answer[0]     )     )

		assert tokenizer.decode(answer[0]     ) == "[UNK] 1 [SEP]"
		if pytorch_dump_folder_path is not None:
				hf_vqa_model.save_pretrained(pytorch_dump_folder_path + '''_vqa'''     )

		SCREAMING_SNAKE_CASE_    			=			'''https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth'''

		SCREAMING_SNAKE_CASE_    			=			blip_itm(pretrained=__a,				image_size=__a,				vit='''base'''     )
		itm_model.eval()

		SCREAMING_SNAKE_CASE_    			=			itm_model.state_dict()
		for key in modified_state_dict.copy():
				SCREAMING_SNAKE_CASE_    			=			modified_state_dict.pop(__a     )
				SCREAMING_SNAKE_CASE_    			=			rename_key(__a     )
				SCREAMING_SNAKE_CASE_    			=			value

		SCREAMING_SNAKE_CASE_    			=			BlipForImageTextRetrieval(__a     )

		SCREAMING_SNAKE_CASE_    			=			['''A picture of a woman with a dog sitting in a beach''']
		SCREAMING_SNAKE_CASE_    			=			tokenizer(
		    __a,				return_tensors='''pt''',				padding='''max_length''',				truncation=__a,				max_length=35,				).input_ids

		hf_itm_model.load_state_dict(__a     )
		hf_itm_model.eval()

		SCREAMING_SNAKE_CASE_    			=			hf_itm_model(__a,				__a,				use_itm_head=__a     )
		SCREAMING_SNAKE_CASE_    			=			hf_itm_model(__a,				__a,				use_itm_head=__a     )

		assert out[0].item() == 0.2_1_1_0_6_8_7_4_9_4_2_7_7_9_5_4
		assert torch.nn.functional.softmax(out_itm[0],				dim=1     )[:, 1].item() == 0.4_5_6_9_8_8_4_5_3_8_6_5_0_5_1_2_7

		if pytorch_dump_folder_path is not None:
				hf_itm_model.save_pretrained(pytorch_dump_folder_path + '''_itm'''     )


if __name__ == "__main__":
						lowerCAmelCase__				=					argparse.ArgumentParser()
						parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
						parser.add_argument('--config_path', default=None, type=str, help='Path to hf config.json of model to convert')
						lowerCAmelCase__				=					parser.parse_args()

						convert_blip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)