

'''simple docstring'''



import os
from pathlib import Path

import numpy as np
import pytest
from pack_dataset import pack_data_dir
from parameterized import parameterized
from save_len_file import save_len_file
from torch.utils.data import DataLoader

from transformers import AutoTokenizer
from transformers.models.mbart.modeling_mbart import shift_tokens_right
from transformers.testing_utils import TestCasePlus, slow
from utils import FAIRSEQ_AVAILABLE, DistributedSortishSampler, LegacySeqaSeqDataset, SeqaSeqDataset


UpperCAmelCase					:			List[str]       =	'bert-base-cased'
UpperCAmelCase					:			Tuple       =	'google/pegasus-xsum'
UpperCAmelCase					:			Optional[int]       =	[' Sam ate lunch today.', 'Sams lunch ingredients.']
UpperCAmelCase					:			Any       =	['A very interesting story about what I ate for lunch.', 'Avocado, celery, turkey, coffee']
UpperCAmelCase					:			List[str]       =	'patrickvonplaten/t5-tiny-random'
UpperCAmelCase					:			Dict       =	'sshleifer/bart-tiny-random'
UpperCAmelCase					:			Optional[Any]       =	'sshleifer/tiny-mbart'
UpperCAmelCase					:			Optional[int]       =	'sshleifer/tiny-marian-en-de'







def   a__			(    a__ ,       a__		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = """\n""".join(a__		)
     Path(a__		).open("""w"""		).writelines(a__		)







def   a__			(    a__		):




     """simple docstring"""


     for split in ["train", "val", "test"]:
          _dump_articles(os.path.join(a__ ,       F'{split}.source'		) ,       a__		)
          _dump_articles(os.path.join(a__ ,       F'{split}.target'		) ,       a__		)
     return tmp_dir







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 @parameterized.expand(
     [
         MBART_TINY,
         MARIAN_TINY,
         T5_TINY,
         BART_TINY,
         PEGASUS_XSUM,
     ]   ,				)
 @slow
 def  UpperCAmelCase__						(						self   :			Union[str, Any]   ,				__SCREAMING_SNAKE_CASE   :			List[str]     )    ->					Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )
      __SCREAMING_SNAKE_CASE      = max(len(tokenizer.encode(__SCREAMING_SNAKE_CASE     )     ) for a in ARTICLES     )
      __SCREAMING_SNAKE_CASE      = max(len(tokenizer.encode(__SCREAMING_SNAKE_CASE     )     ) for a in SUMMARIES     )
      __SCREAMING_SNAKE_CASE      = 4
      __SCREAMING_SNAKE_CASE      = 8
      assert max_len_target > max_src_len  # Will be truncated
      assert max_len_source > max_src_len  # Will be truncated
      __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = """ro_RO""", """de_DE"""  # ignored for all but mbart, but never causes error.
      __SCREAMING_SNAKE_CASE      = SeqaSeqDataset(
          __SCREAMING_SNAKE_CASE   ,				data_dir=__SCREAMING_SNAKE_CASE   ,				type_path="""train"""   ,				max_source_length=__SCREAMING_SNAKE_CASE   ,				max_target_length=__SCREAMING_SNAKE_CASE   ,				src_lang=__SCREAMING_SNAKE_CASE   ,				tgt_lang=__SCREAMING_SNAKE_CASE   ,				)
      __SCREAMING_SNAKE_CASE      = DataLoader(__SCREAMING_SNAKE_CASE   ,				batch_size=2   ,				collate_fn=train_dataset.collate_fn     )
      for batch in dataloader:
           assert isinstance(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )
           assert batch["attention_mask"].shape == batch["input_ids"].shape
           # show that articles were trimmed.
           assert batch["input_ids"].shape[1] == max_src_len
           # show that targets are the same len
           assert batch["labels"].shape[1] == max_tgt_len
           if tok_name != MBART_TINY:
                continue
           # check language codes in correct place
           __SCREAMING_SNAKE_CASE      = shift_tokens_right(batch["""labels"""]   ,				tokenizer.pad_token_id     )
           assert batch["decoder_input_ids"][0, 0].item() == tokenizer.lang_code_to_id[tgt_lang]
           assert batch["decoder_input_ids"][0, -1].item() == tokenizer.eos_token_id
           assert batch["input_ids"][0, -2].item() == tokenizer.eos_token_id
           assert batch["input_ids"][0, -1].item() == tokenizer.lang_code_to_id[src_lang]

           break  # No need to test every batch
 @parameterized.expand([BART_TINY, BERT_BASE_CASED]     )
 def  UpperCAmelCase__						(						self   :			Union[str, Any]   ,				__SCREAMING_SNAKE_CASE   :			Tuple     )    ->					Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )
      __SCREAMING_SNAKE_CASE      = max(len(tokenizer.encode(__SCREAMING_SNAKE_CASE     )     ) for a in ARTICLES     )
      __SCREAMING_SNAKE_CASE      = max(len(tokenizer.encode(__SCREAMING_SNAKE_CASE     )     ) for a in SUMMARIES     )
      __SCREAMING_SNAKE_CASE      = 4
      __SCREAMING_SNAKE_CASE      = LegacySeqaSeqDataset(
          __SCREAMING_SNAKE_CASE   ,				data_dir=__SCREAMING_SNAKE_CASE   ,				type_path="""train"""   ,				max_source_length=20   ,				max_target_length=__SCREAMING_SNAKE_CASE   ,				)
      __SCREAMING_SNAKE_CASE      = DataLoader(__SCREAMING_SNAKE_CASE   ,				batch_size=2   ,				collate_fn=train_dataset.collate_fn     )
      for batch in dataloader:
           assert batch["attention_mask"].shape == batch["input_ids"].shape
           # show that articles were trimmed.
           assert batch["input_ids"].shape[1] == max_len_source
           assert 20 >= batch["input_ids"].shape[1]  # trimmed significantly
           # show that targets were truncated
           assert batch["labels"].shape[1] == trunc_target  # Truncated
           assert max_len_target > trunc_target  # Truncated
           break  # No need to test every batch
 def  UpperCAmelCase__						(						self   :			int     )    ->					Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = AutoTokenizer.from_pretrained("""facebook/mbart-large-cc25"""     )

      __SCREAMING_SNAKE_CASE      = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )     )
      __SCREAMING_SNAKE_CASE      = tmp_dir.joinpath("""train.source"""     ).open().readlines()
      __SCREAMING_SNAKE_CASE      = Path(make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )     )
      pack_data_dir(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE   ,				128   ,				__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = {x.name for x in tmp_dir.iterdir()}
      __SCREAMING_SNAKE_CASE      = {x.name for x in save_dir.iterdir()}
      __SCREAMING_SNAKE_CASE      = save_dir.joinpath("""train.source"""     ).open().readlines()
      # orig: [' Sam ate lunch today.\n', 'Sams lunch ingredients.']
      # desired_packed: [' Sam ate lunch today.\n Sams lunch ingredients.']
      assert len(__SCREAMING_SNAKE_CASE     ) < len(__SCREAMING_SNAKE_CASE     )
      assert len(__SCREAMING_SNAKE_CASE     ) == 1
      assert len(packed_examples[0]     ) == sum(len(__SCREAMING_SNAKE_CASE     ) for x in orig_examples     )
      assert orig_paths == new_paths
 @pytest.mark.skipif(not FAIRSEQ_AVAILABLE   ,				reason="""This test requires fairseq"""     )
 def  UpperCAmelCase__						(						self   :			Optional[Any]     )    ->					List[Any]:


      """simple docstring"""




      if not FAIRSEQ_AVAILABLE:
           return
      __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = self._get_dataset(max_len=64     )
      __SCREAMING_SNAKE_CASE      = 64
      __SCREAMING_SNAKE_CASE      = ds.make_dynamic_sampler(__SCREAMING_SNAKE_CASE   ,				required_batch_size_multiple=__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = [len(__SCREAMING_SNAKE_CASE     ) for x in batch_sampler]
      assert len(set(__SCREAMING_SNAKE_CASE     )     ) > 1  # it's not dynamic batch size if every batch is the same length
      assert sum(__SCREAMING_SNAKE_CASE     ) == len(__SCREAMING_SNAKE_CASE     )  # no dropped or added examples
      __SCREAMING_SNAKE_CASE      = DataLoader(__SCREAMING_SNAKE_CASE   ,				batch_sampler=__SCREAMING_SNAKE_CASE   ,				collate_fn=ds.collate_fn   ,				num_workers=2     )
      __SCREAMING_SNAKE_CASE      = []
      __SCREAMING_SNAKE_CASE      = []
      for batch in data_loader:
           __SCREAMING_SNAKE_CASE      = batch["""input_ids"""].shape
           __SCREAMING_SNAKE_CASE      = src_shape[0]
           assert bs % required_batch_size_multiple == 0 or bs < required_batch_size_multiple
           __SCREAMING_SNAKE_CASE      = np.product(batch["""input_ids"""].shape     )
           num_src_per_batch.append(__SCREAMING_SNAKE_CASE     )
           if num_src_tokens > (max_tokens * 1.1):
                failures.append(__SCREAMING_SNAKE_CASE     )
      assert num_src_per_batch[0] == max(__SCREAMING_SNAKE_CASE     )
      if failures:
           raise AssertionError(f'too many tokens in {len(__SCREAMING_SNAKE_CASE     )} batches'     )
 def  UpperCAmelCase__						(						self   :			Tuple     )    ->					int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = self._get_dataset(max_len=512     )
      __SCREAMING_SNAKE_CASE      = 2
      __SCREAMING_SNAKE_CASE      = ds.make_sortish_sampler(__SCREAMING_SNAKE_CASE   ,				shuffle=__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = DataLoader(__SCREAMING_SNAKE_CASE   ,				batch_size=__SCREAMING_SNAKE_CASE   ,				collate_fn=ds.collate_fn   ,				num_workers=2     )
      __SCREAMING_SNAKE_CASE      = DataLoader(__SCREAMING_SNAKE_CASE   ,				batch_size=__SCREAMING_SNAKE_CASE   ,				collate_fn=ds.collate_fn   ,				num_workers=2   ,				sampler=__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = tokenizer.pad_token_id

      def count_pad_tokens(__SCREAMING_SNAKE_CASE   :			Dict   ,				__SCREAMING_SNAKE_CASE   :			int="input_ids"     ):
           return [batch[k].eq(__SCREAMING_SNAKE_CASE     ).sum().item() for batch in data_loader]

      assert sum(count_pad_tokens(__SCREAMING_SNAKE_CASE   ,				k="""labels"""     )     ) < sum(count_pad_tokens(__SCREAMING_SNAKE_CASE   ,				k="""labels"""     )     )
      assert sum(count_pad_tokens(__SCREAMING_SNAKE_CASE     )     ) < sum(count_pad_tokens(__SCREAMING_SNAKE_CASE     )     )
      assert len(__SCREAMING_SNAKE_CASE     ) == len(__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Union[str, Any]   ,				__SCREAMING_SNAKE_CASE   :			Tuple=1_000   ,				__SCREAMING_SNAKE_CASE   :			int=128     )    ->					List[str]:


      """simple docstring"""




      if os.getenv("""USE_REAL_DATA"""   ,				__SCREAMING_SNAKE_CASE     ):
           __SCREAMING_SNAKE_CASE      = """examples/seq2seq/wmt_en_ro"""
           __SCREAMING_SNAKE_CASE      = max_len * 2 * 64
           if not Path(__SCREAMING_SNAKE_CASE     ).joinpath("""train.len"""     ).exists():
                save_len_file(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )
      else:
           __SCREAMING_SNAKE_CASE      = """examples/seq2seq/test_data/wmt_en_ro"""
           __SCREAMING_SNAKE_CASE      = max_len * 4
           save_len_file(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = SeqaSeqDataset(
          __SCREAMING_SNAKE_CASE   ,				data_dir=__SCREAMING_SNAKE_CASE   ,				type_path="""train"""   ,				max_source_length=__SCREAMING_SNAKE_CASE   ,				max_target_length=__SCREAMING_SNAKE_CASE   ,				n_obs=__SCREAMING_SNAKE_CASE   ,				)
      return ds, max_tokens, tokenizer
 def  UpperCAmelCase__						(						self   :			Any     )    ->					Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = self._get_dataset()
      __SCREAMING_SNAKE_CASE      = set(DistributedSortishSampler(__SCREAMING_SNAKE_CASE   ,				256   ,				num_replicas=2   ,				rank=0   ,				add_extra_examples=__SCREAMING_SNAKE_CASE     )     )
      __SCREAMING_SNAKE_CASE      = set(DistributedSortishSampler(__SCREAMING_SNAKE_CASE   ,				256   ,				num_replicas=2   ,				rank=1   ,				add_extra_examples=__SCREAMING_SNAKE_CASE     )     )
      assert idsa.intersection(__SCREAMING_SNAKE_CASE     ) == set()


 @parameterized.expand(
     [
         MBART_TINY,
         MARIAN_TINY,
         T5_TINY,
         BART_TINY,
         PEGASUS_XSUM,
     ]   ,				)
 def  UpperCAmelCase__						(						self   :			str   ,				__SCREAMING_SNAKE_CASE   :			Tuple     )    ->					Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE   ,				use_fast=__SCREAMING_SNAKE_CASE     )
      if tok_name == MBART_TINY:
           __SCREAMING_SNAKE_CASE      = SeqaSeqDataset(
               __SCREAMING_SNAKE_CASE   ,				data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )   ,				type_path="""train"""   ,				max_source_length=4   ,				max_target_length=8   ,				src_lang="""EN"""   ,				tgt_lang="""FR"""   ,				)
           __SCREAMING_SNAKE_CASE      = train_dataset.dataset_kwargs
           assert "src_lang" in kwargs and "tgt_lang" in kwargs
      else:
           __SCREAMING_SNAKE_CASE      = SeqaSeqDataset(
               __SCREAMING_SNAKE_CASE   ,				data_dir=make_test_data_dir(tmp_dir=self.get_auto_remove_tmp_dir()     )   ,				type_path="""train"""   ,				max_source_length=4   ,				max_target_length=8   ,				)
           __SCREAMING_SNAKE_CASE      = train_dataset.dataset_kwargs
           assert "add_prefix_space" not in kwargs if tok_name != BART_TINY else "add_prefix_space" in kwargs
           assert len(__SCREAMING_SNAKE_CASE     ) == 1 if tok_name == BART_TINY else len(__SCREAMING_SNAKE_CASE     ) == 0









'''simple docstring'''



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase					:			int       =	get_tests_dir('fixtures/test_sentencepiece.model')







@require_sentencepiece
class 		lowerCAmelCase__  ( a					,   unittest.TestCase   ):



 """simple docstring"""



 lowerCAmelCase__       = XLMProphetNetTokenizer
 lowerCAmelCase__       = False
 lowerCAmelCase__       = True
 def  UpperCAmelCase__						(						self   :			str     )    ->					Optional[int]:


      """simple docstring"""




      super().setUp()

      # We have a SentencePiece fixture for testing
      __SCREAMING_SNAKE_CASE      = XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE   ,				keep_accents=__SCREAMING_SNAKE_CASE     )
      tokenizer.save_pretrained(self.tmpdirname     )
 def  UpperCAmelCase__						(						self   :			Dict     )    ->					Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = """[PAD]"""
      __SCREAMING_SNAKE_CASE      = 0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE     )   ,				__SCREAMING_SNAKE_CASE     )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE     )   ,				__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Tuple     )    ->					Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = list(self.get_tokenizer().get_vocab().keys()     )

      self.assertEqual(vocab_keys[0]   ,				"""[PAD]"""     )
      self.assertEqual(vocab_keys[1]   ,				"""[CLS]"""     )
      self.assertEqual(vocab_keys[-1]   ,				"""j"""     )
      self.assertEqual(len(__SCREAMING_SNAKE_CASE     )   ,				1_012     )
 def  UpperCAmelCase__						(						self   :			Any     )    ->					List[str]:


      """simple docstring"""




      self.assertEqual(self.get_tokenizer().vocab_size   ,				1_012     )
 def  UpperCAmelCase__						(						self   :			int     )    ->					List[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE   ,				keep_accents=__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = tokenizer.tokenize("""This is a test"""     )
      self.assertListEqual(__SCREAMING_SNAKE_CASE   ,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]     )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE     )   ,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,				)

      __SCREAMING_SNAKE_CASE      = tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE   ,				[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """9""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """é""",
              """.""",
          ]   ,				)
      __SCREAMING_SNAKE_CASE      = tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE     )
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE   ,				[
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
          ]   ,				)

      __SCREAMING_SNAKE_CASE      = tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE     )
      self.assertListEqual(
          __SCREAMING_SNAKE_CASE   ,				[
              SPIECE_UNDERLINE + """I""",
              SPIECE_UNDERLINE + """was""",
              SPIECE_UNDERLINE + """b""",
              """or""",
              """n""",
              SPIECE_UNDERLINE + """in""",
              SPIECE_UNDERLINE + """""",
              """[UNK]""",
              """2""",
              """0""",
              """0""",
              """0""",
              """,""",
              SPIECE_UNDERLINE + """and""",
              SPIECE_UNDERLINE + """this""",
              SPIECE_UNDERLINE + """is""",
              SPIECE_UNDERLINE + """f""",
              """al""",
              """s""",
              """[UNK]""",
              """.""",
          ]   ,				)
 @cached_property
 def  UpperCAmelCase__						(						self   :			str     )    ->					str:


      """simple docstring"""




      return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""     )
 @slow
 def  UpperCAmelCase__						(						self   :			Dict     )    ->					Optional[Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = """Hello World!"""
      __SCREAMING_SNAKE_CASE      = [35_389, 6_672, 49, 2]
      self.assertListEqual(__SCREAMING_SNAKE_CASE   ,				self.big_tokenizer.encode(__SCREAMING_SNAKE_CASE     )     )


 @slow
 def  UpperCAmelCase__						(						self   :			Dict     )    ->					str:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = {"""input_ids""": [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=__SCREAMING_SNAKE_CASE   ,				model_name="""microsoft/xprophetnet-large-wiki100-cased"""   ,				revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""   ,				)






