





"""simple docstring"""





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 		lowercase					(					_UpperCAmelCase     ,  unittest.TestCase					):
      _SCREAMING_SNAKE_CASE					=      BertJapaneseTokenizer
      _SCREAMING_SNAKE_CASE					=      False
      _SCREAMING_SNAKE_CASE					=      True





      def     _snake_case			(       self )		->     Optional[int]:
            super().setUp()

            lowerCAmelCase  						=       [
                """[UNK]""",
                """[CLS]""",
                """[SEP]""",
                """こんにちは""",
                """こん""",
                """にちは""",
                """ばんは""",
                """##こん""",
                """##にちは""",
                """##ばんは""",
                """世界""",
                """##世界""",
                """、""",
                """##、""",
                """。""",
                """##。""",
            ]

            lowerCAmelCase  						=       os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES["""vocab_file"""] )
            with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8""" ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )





      def     _snake_case			(       self      ,						lowercase )		->     Optional[int]:
            lowerCAmelCase  						=       """こんにちは、世界。 \nこんばんは、世界。"""
            lowerCAmelCase  						=       """こんにちは 、 世界 。 こんばんは 、 世界 。"""
            return input_text, output_text





      def     _snake_case			(       self      ,						lowercase )		->     int:
            lowerCAmelCase						,		lowerCAmelCase  						=       self.get_input_output_texts(lowercase )
            lowerCAmelCase  						=       tokenizer.encode(lowercase      ,						add_special_tokens=lowercase )
            lowerCAmelCase  						=       tokenizer.decode(lowercase      ,						clean_up_tokenization_spaces=lowercase )
            return text, ids





      def     _snake_case			(       self )		->     Tuple:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     Any:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     int:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     Any:
            lowerCAmelCase  						=       self.tokenizer_class(self.vocab_file )

            lowerCAmelCase  						=       tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""" )
            self.assertListEqual(lowercase      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""] )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase )      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )





      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""mecab""" )
            self.assertIsNotNone(lowercase )

            lowerCAmelCase  						=       """こんにちは、世界。\nこんばんは、世界。"""
            lowerCAmelCase  						=       tokenizer.tokenize(lowercase )
            self.assertListEqual(lowercase      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""] )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase )      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

            lowerCAmelCase  						=       os.path.join(self.tmpdirname      ,						"""tokenizer.bin""" )
            with open(lowercase      ,						"""wb""" ) as handle:
                  pickle.dump(lowercase      ,						lowercase )

            with open(lowercase      ,						"""rb""" ) as handle:
                  lowerCAmelCase  						=       pickle.load(lowercase )

            lowerCAmelCase  						=       tokenizer_new.tokenize(lowercase )

            self.assertListEqual(lowercase      ,						lowercase )





      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       MecabTokenizer(mecab_dic="""ipadic""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)





      def     _snake_case			(       self )		->     List[Any]:
            try:
                  lowerCAmelCase  						=       MecabTokenizer(mecab_dic="""unidic_lite""" )
            except ModuleNotFoundError:
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)





      def     _snake_case			(       self )		->     Any:
            try:
                  lowerCAmelCase  						=       MecabTokenizer(mecab_dic="""unidic""" )
            except ModuleNotFoundError:
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)





      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       MecabTokenizer(do_lower_case=lowercase      ,						mecab_dic="""ipadic""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)





      def     _snake_case			(       self )		->     Union[str, Any]:
            try:
                  lowerCAmelCase  						=       MecabTokenizer(
                      do_lower_case=lowercase      ,						normalize_text=lowercase      ,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""" )
            except RuntimeError:
                  # if dict doesn't exist in the system, previous code raises this error.
                  return

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)





      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       MecabTokenizer(normalize_text=lowercase      ,						mecab_dic="""ipadic""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]      ,						)





      @require_sudachi
      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""sudachi""" )
            self.assertIsNotNone(lowercase )

            lowerCAmelCase  						=       """こんにちは、世界。\nこんばんは、世界。"""
            lowerCAmelCase  						=       tokenizer.tokenize(lowercase )
            self.assertListEqual(lowercase      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""] )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase )      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

            lowerCAmelCase  						=       os.path.join(self.tmpdirname      ,						"""tokenizer.bin""" )
            with open(lowercase      ,						"""wb""" ) as handle:
                  pickle.dump(lowercase      ,						lowercase )

            with open(lowercase      ,						"""rb""" ) as handle:
                  lowerCAmelCase  						=       pickle.load(lowercase )

            lowerCAmelCase  						=       tokenizer_new.tokenize(lowercase )

            self.assertListEqual(lowercase      ,						lowercase )





      @require_sudachi
      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       SudachiTokenizer(sudachi_dict_type="""core""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,						)





      @require_sudachi
      def     _snake_case			(       self )		->     Any:
            lowerCAmelCase  						=       SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""A""" )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権""" )      ,						["""外国""", """人""", """参政""", """権"""] )





      @require_sudachi
      def     _snake_case			(       self )		->     Optional[Any]:
            lowerCAmelCase  						=       SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""B""" )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権""" )      ,						["""外国人""", """参政権"""] )





      @require_sudachi
      def     _snake_case			(       self )		->     Optional[Any]:
            lowerCAmelCase  						=       SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""C""" )

            self.assertListEqual(tokenizer.tokenize("""外国人参政権""" )      ,						["""外国人参政権"""] )





      @require_sudachi
      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase  						=       SudachiTokenizer(do_lower_case=lowercase      ,						sudachi_dict_type="""core""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,						)





      @require_sudachi
      def     _snake_case			(       self )		->     Union[str, Any]:
            lowerCAmelCase  						=       SudachiTokenizer(normalize_text=lowercase      ,						sudachi_dict_type="""core""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]      ,						)





      @require_sudachi
      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       SudachiTokenizer(trim_whitespace=lowercase      ,						sudachi_dict_type="""core""" )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)





      @require_jumanpp
      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""jumanpp""" )
            self.assertIsNotNone(lowercase )

            lowerCAmelCase  						=       """こんにちは、世界。\nこんばんは、世界。"""
            lowerCAmelCase  						=       tokenizer.tokenize(lowercase )
            self.assertListEqual(lowercase      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""] )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase )      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14] )

            lowerCAmelCase  						=       os.path.join(self.tmpdirname      ,						"""tokenizer.bin""" )
            with open(lowercase      ,						"""wb""" ) as handle:
                  pickle.dump(lowercase      ,						lowercase )

            with open(lowercase      ,						"""rb""" ) as handle:
                  lowerCAmelCase  						=       pickle.load(lowercase )

            lowerCAmelCase  						=       tokenizer_new.tokenize(lowercase )

            self.assertListEqual(lowercase      ,						lowercase )





      @require_jumanpp
      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)





      @require_jumanpp
      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase  						=       JumanppTokenizer(do_lower_case=lowercase )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)





      @require_jumanpp
      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       JumanppTokenizer(normalize_text=lowercase )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)





      @require_jumanpp
      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       JumanppTokenizer(trim_whitespace=lowercase )

            self.assertListEqual(
                tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """ )      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]      ,						)





      @require_jumanpp
      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""" )      ,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]      ,						)





      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

            lowerCAmelCase  						=       {}
            for i, token in enumerate(lowercase ):
                  lowerCAmelCase  						=       i
            lowerCAmelCase  						=       WordpieceTokenizer(vocab=lowercase      ,						unk_token="""[UNK]""" )

            self.assertListEqual(tokenizer.tokenize("""""" )      ,						[] )

            self.assertListEqual(tokenizer.tokenize("""こんにちは""" )      ,						["""こんにちは"""] )

            self.assertListEqual(tokenizer.tokenize("""こんばんは""" )      ,						["""こん""", """##ばんは"""] )

            self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""" )      ,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""] )





      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""" )
            lowerCAmelCase  						=       tokenizer.subword_tokenizer

            lowerCAmelCase  						=       subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""" )
            self.assertListEqual(lowercase      ,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""] )

            lowerCAmelCase  						=       subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""" )
            self.assertListEqual(lowercase      ,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""] )





      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""" )

            lowerCAmelCase  						=       tokenizer.encode("""ありがとう。"""      ,						add_special_tokens=lowercase )
            lowerCAmelCase  						=       tokenizer.encode("""どういたしまして。"""      ,						add_special_tokens=lowercase )

            lowerCAmelCase  						=       tokenizer.build_inputs_with_special_tokens(lowercase )
            lowerCAmelCase  						=       tokenizer.build_inputs_with_special_tokens(lowercase      ,						lowercase )

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		lowercase					(					_UpperCAmelCase     ,  unittest.TestCase					):
      _SCREAMING_SNAKE_CASE					=      BertJapaneseTokenizer
      _SCREAMING_SNAKE_CASE					=      False





      def     _snake_case			(       self )		->     Optional[int]:
            super().setUp()

            lowerCAmelCase  						=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

            lowerCAmelCase  						=       os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES["""vocab_file"""] )
            with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8""" ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens] ) )





      def     _snake_case			(       self      ,						**lowercase )		->     Optional[int]:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,						subword_tokenizer_type="""character"""      ,						**lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     Union[str, Any]:
            lowerCAmelCase  						=       """こんにちは、世界。 \nこんばんは、世界。"""
            lowerCAmelCase  						=       """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
            return input_text, output_text





      def     _snake_case			(       self )		->     Dict:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     Tuple:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     Tuple:
            pass  # TODO add if relevant





      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       self.tokenizer_class(self.vocab_file      ,						subword_tokenizer_type="""character""" )

            lowerCAmelCase  						=       tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""" )
            self.assertListEqual(
                lowercase      ,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""] )
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(lowercase )      ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12] )





      def     _snake_case			(       self )		->     Optional[int]:
            lowerCAmelCase  						=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

            lowerCAmelCase  						=       {}
            for i, token in enumerate(lowercase ):
                  lowerCAmelCase  						=       i
            lowerCAmelCase  						=       CharacterTokenizer(vocab=lowercase      ,						unk_token="""[UNK]""" )

            self.assertListEqual(tokenizer.tokenize("""""" )      ,						[] )

            self.assertListEqual(tokenizer.tokenize("""こんにちは""" )      ,						["""こ""", """ん""", """に""", """ち""", """は"""] )

            self.assertListEqual(tokenizer.tokenize("""こんにちほ""" )      ,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""] )





      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""" )

            lowerCAmelCase  						=       tokenizer.encode("""ありがとう。"""      ,						add_special_tokens=lowercase )
            lowerCAmelCase  						=       tokenizer.encode("""どういたしまして。"""      ,						add_special_tokens=lowercase )

            lowerCAmelCase  						=       tokenizer.build_inputs_with_special_tokens(lowercase )
            lowerCAmelCase  						=       tokenizer.build_inputs_with_special_tokens(lowercase      ,						lowercase )

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		lowercase					(					unittest.TestCase					):





      def     _snake_case			(       self )		->     Union[str, Any]:
            lowerCAmelCase  						=       """cl-tohoku/bert-base-japanese"""
            lowerCAmelCase  						=       AutoTokenizer.from_pretrained(lowercase )
            self.assertIsInstance(lowercase      ,						lowercase )







class 		lowercase					(					unittest.TestCase					):





      def     _snake_case			(       self )		->     Dict:
            lowerCAmelCase  						=       """cl-tohoku/bert-base-japanese"""
            with self.assertLogs("""transformers"""      ,						level="""WARNING""" ) as cm:
                  BertTokenizer.from_pretrained(lowercase )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from.""" ) )
            lowerCAmelCase  						=       """bert-base-cased"""
            with self.assertLogs("""transformers"""      ,						level="""WARNING""" ) as cm:
                  BertJapaneseTokenizer.from_pretrained(lowercase )
                  self.assertTrue(
                      cm.records[0].message.startswith(
                          """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                          """ is called from.""" ) )










import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch







class  snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__					:     Tuple    =							JukeboxTokenizer
						SCREAMING_SNAKE_CASE__					:     int    =							{
						    """artist""": """Zac Brown Band""",
						    """genres""": """Country""",
						    """lyrics""": """I met a traveller from an antique land,
        Who said \"Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        """,
						}




						@require_torch
						def   lowercase_       ( self							)    ->			Union[str, Any]:
										import torch

										lowerCAmelCase_       :					Union[str, Any]			 = JukeboxTokenizer.from_pretrained('''openai/jukebox-1b-lyrics'''							)
										lowerCAmelCase_       :					Any			 = tokenizer(**self.metas							)['''input_ids''']
										# fmt: off
										lowerCAmelCase_       :					List[str]			 = [
										    torch.tensor([[
										        0, 0, 0, 7_1_6_9, 5_0_7, 9, 7_6, 3_9, 3_1, 4_6, 7_6, 2_7,
										        7_6, 4_6, 4_4, 2_7, 4_8, 3_1, 3_8, 3_8, 3_1, 4_4, 7_6, 3_2,
										        4_4, 4_1, 3_9, 7_6, 2_7, 4_0, 7_6, 2_7, 4_0, 4_6, 3_5, 4_3,
										        4_7, 3_1, 7_6, 3_8, 2_7, 4_0, 3_0, 6_4, 7_8, 7_6, 7_6, 7_6,
										        7_6, 7_6, 7_6, 7_6, 7_6, 2_3, 3_4, 4_1, 7_6, 4_5, 2_7, 3_5,
										        3_0, 7_6, 7_1, 2_0, 4_9, 4_1, 7_6, 4_8, 2_7, 4_5, 4_6, 7_6,
										        2_7, 4_0, 3_0, 7_6, 4_6, 4_4, 4_7, 4_0, 3_7, 3_8, 3_1, 4_5,
										        4_5, 7_6, 3_8, 3_1, 3_3, 4_5, 7_6, 4_1, 3_2, 7_6, 4_5, 4_6,
										        4_1, 4_0, 3_1, 7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6,
										        1_9, 4_6, 2_7, 4_0, 3_0, 7_6, 3_5, 4_0, 7_6, 4_6, 3_4, 3_1,
										        7_6, 3_0, 3_1, 4_5, 3_1, 4_4, 4_6, 6_3, 7_6, 6_3, 7_6, 6_3,
										        7_6, 6_3, 7_6, 1_4, 3_1, 2_7, 4_4, 7_6, 4_6, 3_4, 3_1, 3_9,
										        6_4, 7_6, 4_1, 4_0, 7_6, 4_6, 3_4, 3_1, 7_6, 4_5, 2_7, 4_0,
										        3_0, 6_4, 7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 8,
										        2_7, 3_8, 3_2, 7_6, 4_5, 4_7, 4_0, 3_7, 7_6, 2_7, 7_6, 4_5,
										        3_4, 2_7, 4_6, 4_6, 3_1, 4_4, 3_1, 3_0, 7_6, 4_8, 3_5, 4_5,
										        2_7, 3_3, 3_1, 7_6, 3_8, 3_5, 3_1, 4_5, 6_4, 7_6, 4_9, 3_4,
										        4_1, 4_5, 3_1, 7_6, 3_2, 4_4, 4_1, 4_9, 4_0, 6_4, 7_8, 7_6,
										        7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1, 4_0, 3_0, 7_6, 4_9,
										        4_4, 3_5, 4_0, 3_7, 3_8, 3_1, 3_0, 7_6, 3_8, 3_5, 4_2, 6_4,
										        7_6, 2_7, 4_0, 3_0, 7_6, 4_5, 4_0, 3_1, 3_1, 4_4, 7_6, 4_1,
										        3_2, 7_6, 2_9, 4_1, 3_8, 3_0, 7_6, 2_9, 4_1, 3_9, 3_9, 2_7,
										        4_0, 3_0, 6_4, 7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6,
										        2_0, 3_1, 3_8, 3_8, 7_6, 4_6, 3_4, 2_7, 4_6, 7_6, 3_5, 4_6,
										        4_5, 7_6, 4_5, 2_9, 4_7, 3_8, 4_2, 4_6, 4_1, 4_4, 7_6, 4_9,
										        3_1, 3_8, 3_8, 7_6, 4_6, 3_4, 4_1, 4_5, 3_1, 7_6, 4_2, 2_7,
										        4_5, 4_5, 3_5, 4_1, 4_0, 4_5, 7_6, 4_4, 3_1, 2_7, 3_0, 7_8,
										        7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 2_3, 3_4, 3_5, 2_9,
										        3_4, 7_6, 5_1, 3_1, 4_6, 7_6, 4_5, 4_7, 4_4, 4_8, 3_5, 4_8,
										        3_1, 6_4, 7_6, 4_5, 4_6, 2_7, 3_9, 4_2, 3_1, 3_0, 7_6, 4_1,
										        4_0, 7_6, 4_6, 3_4, 3_1, 4_5, 3_1, 7_6, 3_8, 3_5, 3_2, 3_1,
										        3_8, 3_1, 4_5, 4_5, 7_6, 4_6, 3_4, 3_5, 4_0, 3_3, 4_5, 6_4,
										        7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 2_0, 3_4, 3_1,
										        7_6, 3_4, 2_7, 4_0, 3_0, 7_6, 4_6, 3_4, 2_7, 4_6, 7_6, 3_9,
										        4_1, 2_9, 3_7, 3_1, 3_0, 7_6, 4_6, 3_4, 3_1, 3_9, 6_4, 7_6,
										        2_7, 4_0, 3_0, 7_6, 4_6, 3_4, 3_1, 7_6, 3_4, 3_1, 2_7, 4_4,
										        4_6, 7_6, 4_6, 3_4, 2_7, 4_6, 7_6, 3_2, 3_1, 3_0, 6_6, 7_8,
										        7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1, 4_0, 3_0, 7_6,
										        4_1, 4_0, 7_6, 4_6, 3_4, 3_1, 7_6, 4_2, 3_1, 3_0, 3_1, 4_5,
										        4_6, 2_7, 3_8, 6_4, 7_6, 4_6, 3_4, 3_1, 4_5, 3_1, 7_6, 4_9,
										        4_1, 4_4, 3_0, 4_5, 7_6, 2_7, 4_2, 4_2, 3_1, 2_7, 4_4, 6_5,
										        7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1_3, 5_1, 7_6,
										        4_0, 2_7, 3_9, 3_1, 7_6, 3_5, 4_5, 7_6, 1_5, 5_2, 5_1, 3_9,
										        2_7, 4_0, 3_0, 3_5, 2_7, 4_5, 6_4, 7_6, 1_1, 3_5, 4_0, 3_3,
										        7_6, 4_1, 3_2, 7_6, 1_1, 3_5, 4_0, 3_3, 4_5, 6_6, 7_8, 7_6,
										        7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1_2, 4_1, 4_1, 3_7, 7_6,
										        4_1, 4_0, 7_6, 3_9, 5_1, 7_6, 2_3, 4_1, 4_4, 3_7, 4_5, 6_4,
										        7_6, 5_1, 3_1, 7_6, 1_3, 3_5, 3_3, 3_4, 4_6, 5_1, 6_4, 7_6,
										        2_7, 4_0, 3_0, 7_6, 3_0, 3_1, 4_5, 4_2, 2_7, 3_5, 4_4, 6_7,
										        7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1_4, 4_1, 4_6,
										        3_4, 3_5, 4_0, 3_3, 7_6, 2_8, 3_1, 4_5, 3_5, 3_0, 3_1, 7_6,
										        4_4, 3_1, 3_9, 2_7, 3_5, 4_0, 4_5, 6_3, 7_6, 1_8, 4_1, 4_7,
										        4_0, 3_0, 7_6, 4_6, 3_4, 3_1, 7_6, 3_0, 3_1, 2_9, 2_7, 5_1,
										        7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 1_5, 3_2, 7_6,
										        4_6, 3_4, 2_7, 4_6, 7_6, 2_9, 4_1, 3_8, 4_1, 4_5, 4_5, 2_7,
										        3_8, 7_6, 2_3, 4_4, 3_1, 2_9, 3_7, 6_4, 7_6, 2_8, 4_1, 4_7,
										        4_0, 3_0, 3_8, 3_1, 4_5, 4_5, 7_6, 2_7, 4_0, 3_0, 7_6, 2_8,
										        2_7, 4_4, 3_1, 7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6,
										        2_0, 3_4, 3_1, 7_6, 3_8, 4_1, 4_0, 3_1, 7_6, 2_7, 4_0, 3_0,
										        7_6, 3_8, 3_1, 4_8, 3_1, 3_8, 7_6, 4_5, 2_7, 4_0, 3_0, 4_5,
										        7_6, 4_5, 4_6, 4_4, 3_1, 4_6, 2_9, 3_4, 7_6, 3_2, 2_7, 4_4,
										        7_6, 2_7, 4_9, 2_7, 5_1, 7_8, 7_6, 7_6, 7_6, 7_6, 7_6, 7_6,
										        7_6, 7_6]]							),
										    torch.tensor([[0, 0, 0, 1_0_6_9, 1_1]]							),
										    torch.tensor([[0, 0, 0, 1_0_6_9, 1_1]]							),
										]
										# fmt: on
										self.assertTrue(torch.allclose(tokens[0]      ,  EXPECTED_OUTPUT[0]							)							)
										self.assertTrue(torch.allclose(tokens[1]      ,  EXPECTED_OUTPUT[1]							)							)
										self.assertTrue(torch.allclose(tokens[2]      ,  EXPECTED_OUTPUT[2]							)							)




						@require_torch
						def   lowercase_       ( self							)    ->			List[Any]:
										import torch

										lowerCAmelCase_       :					Any			 = JukeboxTokenizer.from_pretrained('''openai/jukebox-5b-lyrics'''							)
										lowerCAmelCase_       :					str			 = tokenizer(**self.metas							)['''input_ids''']
										# fmt: off
										lowerCAmelCase_       :					Tuple			 = [
										    torch.tensor([[
										        0, 0, 0, 1_0_6_9, 1_1, -1, -1, -1, -1, 9, 7_7, 3_9,
										        3_1, 4_6, 7_7, 2_7, 7_7, 4_6, 4_4, 2_7, 4_8, 3_1, 3_8, 3_8,
										        3_1, 4_4, 7_7, 3_2, 4_4, 4_1, 3_9, 7_7, 2_7, 4_0, 7_7, 2_7,
										        4_0, 4_6, 3_5, 4_3, 4_7, 3_1, 7_7, 3_8, 2_7, 4_0, 3_0, 6_4,
										        7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 2_3, 3_4, 4_1,
										        7_7, 4_5, 2_7, 3_5, 3_0, 7_7, 7_2, 2_0, 4_9, 4_1, 7_7, 4_8,
										        2_7, 4_5, 4_6, 7_7, 2_7, 4_0, 3_0, 7_7, 4_6, 4_4, 4_7, 4_0,
										        3_7, 3_8, 3_1, 4_5, 4_5, 7_7, 3_8, 3_1, 3_3, 4_5, 7_7, 4_1,
										        3_2, 7_7, 4_5, 4_6, 4_1, 4_0, 3_1, 7_9, 7_7, 7_7, 7_7, 7_7,
										        7_7, 7_7, 7_7, 7_7, 1_9, 4_6, 2_7, 4_0, 3_0, 7_7, 3_5, 4_0,
										        7_7, 4_6, 3_4, 3_1, 7_7, 3_0, 3_1, 4_5, 3_1, 4_4, 4_6, 6_3,
										        7_7, 6_3, 7_7, 6_3, 7_7, 6_3, 7_7, 1_4, 3_1, 2_7, 4_4, 7_7,
										        4_6, 3_4, 3_1, 3_9, 6_4, 7_7, 4_1, 4_0, 7_7, 4_6, 3_4, 3_1,
										        7_7, 4_5, 2_7, 4_0, 3_0, 6_4, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7,
										        7_7, 7_7, 7_7, 8, 2_7, 3_8, 3_2, 7_7, 4_5, 4_7, 4_0, 3_7,
										        7_7, 2_7, 7_7, 4_5, 3_4, 2_7, 4_6, 4_6, 3_1, 4_4, 3_1, 3_0,
										        7_7, 4_8, 3_5, 4_5, 2_7, 3_3, 3_1, 7_7, 3_8, 3_5, 3_1, 4_5,
										        6_4, 7_7, 4_9, 3_4, 4_1, 4_5, 3_1, 7_7, 3_2, 4_4, 4_1, 4_9,
										        4_0, 6_4, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 1,
										        4_0, 3_0, 7_7, 4_9, 4_4, 3_5, 4_0, 3_7, 3_8, 3_1, 3_0, 7_7,
										        3_8, 3_5, 4_2, 6_4, 7_7, 2_7, 4_0, 3_0, 7_7, 4_5, 4_0, 3_1,
										        3_1, 4_4, 7_7, 4_1, 3_2, 7_7, 2_9, 4_1, 3_8, 3_0, 7_7, 2_9,
										        4_1, 3_9, 3_9, 2_7, 4_0, 3_0, 6_4, 7_9, 7_7, 7_7, 7_7, 7_7,
										        7_7, 7_7, 7_7, 7_7, 2_0, 3_1, 3_8, 3_8, 7_7, 4_6, 3_4, 2_7,
										        4_6, 7_7, 3_5, 4_6, 4_5, 7_7, 4_5, 2_9, 4_7, 3_8, 4_2, 4_6,
										        4_1, 4_4, 7_7, 4_9, 3_1, 3_8, 3_8, 7_7, 4_6, 3_4, 4_1, 4_5,
										        3_1, 7_7, 4_2, 2_7, 4_5, 4_5, 3_5, 4_1, 4_0, 4_5, 7_7, 4_4,
										        3_1, 2_7, 3_0, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        2_3, 3_4, 3_5, 2_9, 3_4, 7_7, 5_1, 3_1, 4_6, 7_7, 4_5, 4_7,
										        4_4, 4_8, 3_5, 4_8, 3_1, 6_4, 7_7, 4_5, 4_6, 2_7, 3_9, 4_2,
										        3_1, 3_0, 7_7, 4_1, 4_0, 7_7, 4_6, 3_4, 3_1, 4_5, 3_1, 7_7,
										        3_8, 3_5, 3_2, 3_1, 3_8, 3_1, 4_5, 4_5, 7_7, 4_6, 3_4, 3_5,
										        4_0, 3_3, 4_5, 6_4, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        7_7, 2_0, 3_4, 3_1, 7_7, 3_4, 2_7, 4_0, 3_0, 7_7, 4_6, 3_4,
										        2_7, 4_6, 7_7, 3_9, 4_1, 2_9, 3_7, 3_1, 3_0, 7_7, 4_6, 3_4,
										        3_1, 3_9, 6_4, 7_7, 2_7, 4_0, 3_0, 7_7, 4_6, 3_4, 3_1, 7_7,
										        3_4, 3_1, 2_7, 4_4, 4_6, 7_7, 4_6, 3_4, 2_7, 4_6, 7_7, 3_2,
										        3_1, 3_0, 6_6, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        1, 4_0, 3_0, 7_7, 4_1, 4_0, 7_7, 4_6, 3_4, 3_1, 7_7, 4_2,
										        3_1, 3_0, 3_1, 4_5, 4_6, 2_7, 3_8, 6_4, 7_7, 4_6, 3_4, 3_1,
										        4_5, 3_1, 7_7, 4_9, 4_1, 4_4, 3_0, 4_5, 7_7, 2_7, 4_2, 4_2,
										        3_1, 2_7, 4_4, 6_5, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        7_7, 1_3, 5_1, 7_7, 4_0, 2_7, 3_9, 3_1, 7_7, 3_5, 4_5, 7_7,
										        1_5, 5_2, 5_1, 3_9, 2_7, 4_0, 3_0, 3_5, 2_7, 4_5, 6_4, 7_7,
										        1_1, 3_5, 4_0, 3_3, 7_7, 4_1, 3_2, 7_7, 1_1, 3_5, 4_0, 3_3,
										        4_5, 6_6, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 1_2,
										        4_1, 4_1, 3_7, 7_7, 4_1, 4_0, 7_7, 3_9, 5_1, 7_7, 2_3, 4_1,
										        4_4, 3_7, 4_5, 6_4, 7_7, 5_1, 3_1, 7_7, 1_3, 3_5, 3_3, 3_4,
										        4_6, 5_1, 6_4, 7_7, 2_7, 4_0, 3_0, 7_7, 3_0, 3_1, 4_5, 4_2,
										        2_7, 3_5, 4_4, 6_7, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        7_7, 1_4, 4_1, 4_6, 3_4, 3_5, 4_0, 3_3, 7_7, 2_8, 3_1, 4_5,
										        3_5, 3_0, 3_1, 7_7, 4_4, 3_1, 3_9, 2_7, 3_5, 4_0, 4_5, 6_3,
										        7_7, 1_8, 4_1, 4_7, 4_0, 3_0, 7_7, 4_6, 3_4, 3_1, 7_7, 3_0,
										        3_1, 2_9, 2_7, 5_1, 7_9, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7, 7_7,
										        7_7, 1_5, 3_2, 7_7, 4_6, 3_4, 2_7, 4_6, 7_7, 2_9, 4_1, 3_8,
										        4_1, 4_5, 4_5, 2_7, 3_8, 7_7, 2_3, 4_4, 3_1, 2_9, 3_7, 6_4,
										        7_7, 2_8, 4_1, 4_7, 4_0, 3_0, 3_8, 3_1, 4_5, 4_5, 7_7, 2_7,
										        4_0, 3_0, 7_7, 2_8, 2_7, 4_4, 3_1, 7_9, 7_7, 7_7, 7_7, 7_7,
										        7_7, 7_7, 7_7, 7_7, 2_0, 3_4, 3_1, 7_7, 3_8, 4_1, 4_0, 3_1,
										        7_7, 2_7, 4_0, 3_0, 7_7, 3_8, 3_1, 4_8, 3_1, 3_8, 7_7, 4_5,
										        2_7, 4_0, 3_0, 4_5, 7_7, 4_5, 4_6, 4_4, 3_1, 4_6, 2_9, 3_4,
										        7_7, 3_2, 2_7, 4_4, 7_7, 2_7, 4_9, 2_7, 5_1, 7_9, 7_7, 7_7,
										        7_7, 7_7, 7_7, 7_7, 7_7, 7_7]]							),
										    torch.tensor([[0, 0, 0, 1_0_6_9, 1_1, -1, -1, -1, -1]]							),
										    torch.tensor([[0, 0, 0, 1_0_6_9, 1_1, -1, -1, -1, -1]]							),
										]
										# fmt: on
										self.assertTrue(torch.allclose(tokens[0]      ,  EXPECTED_OUTPUT[0]							)							)
										self.assertTrue(torch.allclose(tokens[1]      ,  EXPECTED_OUTPUT[1]							)							)
										self.assertTrue(torch.allclose(tokens[2]      ,  EXPECTED_OUTPUT[2]							)							)