'''simple docstring'''






import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
     from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
     __a									=   True
     from torch.cuda.amp import autocast


__a									=   logging.getLogger(__name__)





@dataclass
class        A__				:





   """simple docstring"""





   UpperCamelCase_			:		Optional[Any]       							=	field(
       metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}  )
   UpperCamelCase_			:		Dict       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}   ,						)
   UpperCamelCase_			:		Optional[int]       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}  )
   UpperCamelCase_			:		str       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''Whether to log verbose messages or not.'''}   ,						)
   UpperCamelCase_			:		Tuple       							=	field(
       default=2.0   ,						metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}  )
   UpperCamelCase_			:		str       							=	field(
       default=0.5   ,						metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}  )
   UpperCamelCase_			:		List[Any]       							=	field(
       default=0.99_99_95   ,						metadata={'''help''': '''Decay of gumbel temperature during training.'''}  )






def   __UpperCAmelCase     (    a_:			ModelArguments,   a_:			TrainingArguments				):
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",   datefmt="%m/%d/%Y %H:%M:%S",   handlers=[logging.StreamHandler(sys.stdout				)],   )
   _UpperCAmelCase				:      List[Any]				=  logging.WARNING
   if model_args.verbose_logging:
      _UpperCAmelCase				:      Dict				=  logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank				):
      _UpperCAmelCase				:      Optional[Any]				=  logging.INFO
   logger.setLevel(snake_case__				)





@dataclass
class        A__				:





   """simple docstring"""





   UpperCamelCase_			:		Dict       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}  )
   UpperCamelCase_			:		Optional[Any]       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}  )
   UpperCamelCase_			:		List[str]       							=	field(
       default='''train'''   ,						metadata={
           '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
       }   ,						)
   UpperCamelCase_			:		List[Any]       							=	field(
       default='''validation'''   ,						metadata={
           '''help''': (
               '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
           )
       }   ,						)
   UpperCamelCase_			:		List[Any]       							=	field(
       default='''file'''   ,						metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}   ,						)
   UpperCamelCase_			:		List[str]       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}  )
   UpperCamelCase_			:		str       							=	field(
       default=1   ,						metadata={
           '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
       }   ,						)
   UpperCamelCase_			:		Any       							=	field(
       default=UpperCamelCase   ,						metadata={'''help''': '''The number of processes to use for the preprocessing.'''}   ,						)
   UpperCamelCase_			:		Optional[int]       							=	field(
       default=20.0   ,						metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}  )





@dataclass
class        A__				:





   """simple docstring"""





   UpperCamelCase_			:		Optional[int]       							=	42
   UpperCamelCase_			:		Any       							=	42
   UpperCamelCase_			:		Union[str, Any]       							=	'''longest'''
   UpperCamelCase_			:		List[str]       							=	None
   UpperCamelCase_			:		Any       							=	None




   def __call__(					self	:						List[str]		,	lowerCAmelCase__	:						Optional[int]     )			->       Tuple:







      """simple docstring"""




      _UpperCAmelCase				:      Optional[int]				=  self.feature_extractor.pad(
          A_		,	max_length=self.max_length		,	padding=self.padding		,	pad_to_multiple_of=self.pad_to_multiple_of		,	return_tensors="pt"		,	)
      _UpperCAmelCase				:      str				=  self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]     )

      _UpperCAmelCase				:      Optional[int]				=  batch["""input_values"""].shape[0]

      # make sure that no loss is computed on padded inputs
      if batch["attention_mask"] is not None:
         # compute real output lengths according to convolution formula
         _UpperCAmelCase				:      Tuple				=  self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1     )     ).to(
             torch.long     )

         _UpperCAmelCase				:      Dict				=  torch.zeros(
             (batch_size, mask_indices_seq_length)		,	dtype=torch.long		,	device=batch["input_values"].device     )

         # these two operations makes sure that all values
         # before the output lengths indices are attended to
         _UpperCAmelCase				:      Tuple				=  1
         _UpperCAmelCase				:      int				=  attention_mask.flip([-1]     ).cumsum(-1     ).flip([-1]     ).bool()

      # sample randomly masked indices
      _UpperCAmelCase				:      Dict				=  _compute_mask_indices(
          (batch_size, mask_indices_seq_length)		,	self.model.config.mask_time_prob		,	self.model.config.mask_time_length		,	attention_mask=A_		,	min_masks=2		,	)

      return batch





class        A__				(			UpperCamelCase  ):





   """simple docstring"""





   def __init__(					self	:						Optional[int]		,	*lowerCAmelCase__	:						Any		,	lowerCAmelCase__	:						int=1		,	lowerCAmelCase__	:						Dict=0		,	lowerCAmelCase__	:						Tuple=1.0		,	**lowerCAmelCase__	:						List[Any]     )			->       List[Any]:







      """simple docstring"""




      super().__init__(*A_		,	**A_     )
      _UpperCAmelCase				:      Any				=  0
      _UpperCAmelCase				:      List[str]				=  max_gumbel_temp
      _UpperCAmelCase				:      List[Any]				=  min_gumbel_temp
      _UpperCAmelCase				:      str				=  gumbel_temp_decay




   def      _lowerCAmelCase						(					self	:						Optional[Any]		,	lowerCAmelCase__	:						Union[str, Any]		,	lowerCAmelCase__	:						Tuple     )			->       Any:







      """simple docstring"""




      model.train()
      _UpperCAmelCase				:      int				=  self._prepare_inputs(A_     )

      if self.use_amp:
         with autocast():
            _UpperCAmelCase				:      Union[str, Any]				=  self.compute_loss(A_		,	A_     )
      else:
         _UpperCAmelCase				:      str				=  self.compute_loss(A_		,	A_     )

      if self.args.n_gpu > 1 or self.deepspeed:
         if model.module.config.ctc_loss_reduction == "mean":
            _UpperCAmelCase				:      int				=  loss.mean()
         elif model.module.config.ctc_loss_reduction == "sum":
            _UpperCAmelCase				:      Dict				=  loss.sum() / (inputs["""mask_time_indices"""]).sum()
         else:
            raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']"""     )

      if self.args.gradient_accumulation_steps > 1:
         _UpperCAmelCase				:      Tuple				=  loss / self.args.gradient_accumulation_steps

      if self.use_amp:
         self.scaler.scale(A_     ).backward()
      elif self.use_apex:
         with amp.scale_loss(A_		,	self.optimizer     ) as scaled_loss:
            scaled_loss.backward()
      elif self.deepspeed:
         self.deepspeed.backward(A_     )
      else:
         loss.backward()

      self.num_update_step += 1
      # make sure gumbel softmax temperature is decayed
      if self.args.n_gpu > 1 or self.deepspeed:
         model.module.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,	self.min_gumbel_temp     )     )
      else:
         model.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,	self.min_gumbel_temp     )     )

      return loss.detach()






def   __UpperCAmelCase     (    ):
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   _UpperCAmelCase				:      int				=  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

   _UpperCAmelCase				:      Optional[int]				=  parser.parse_args_into_dataclasses()
   configure_logger(snake_case__,   snake_case__				)

   # Downloading and loading a dataset from the hub.
   _UpperCAmelCase				:      Union[str, Any]				=  load_dataset(data_args.dataset_name,   data_args.dataset_config_name,   cache_dir=model_args.cache_dir				)

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      _UpperCAmelCase				:      Any				=  DatasetDict()
      _UpperCAmelCase				:      Any				=  load_dataset(
          data_args.dataset_name,   data_args.dataset_config_name,   split=f"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]""",   cache_dir=model_args.cache_dir,   )
      _UpperCAmelCase				:      List[Any]				=  load_dataset(
          data_args.dataset_name,   data_args.dataset_config_name,   split=f"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]""",   cache_dir=model_args.cache_dir,   )
   else:
      # make sure only "validation" and "train" keys remain"
      _UpperCAmelCase				:      Optional[Any]				=  DatasetDict()
      _UpperCAmelCase				:      Tuple				=  load_dataset(
          data_args.dataset_name,   data_args.dataset_config_name,   split="validation",   cache_dir=model_args.cache_dir,   )
      _UpperCAmelCase				:      Union[str, Any]				=  load_dataset(
          data_args.dataset_name,   data_args.dataset_config_name,   split=f"""{data_args.train_split_name}""",   cache_dir=model_args.cache_dir,   )

   # only normalized-inputs-training is supported
   _UpperCAmelCase				:      str				=  WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path,   cache_dir=model_args.cache_dir,   do_normalize=snake_case__				)

   def prepare_dataset(a_:			Union[str, Any]				):
      # check that all files have the correct sampling rate
      _UpperCAmelCase				:      Union[str, Any]				=  librosa.load(batch[data_args.speech_file_column],   sr=feature_extractor.sampling_rate				)
      return batch

   # load audio files into numpy arrays
   _UpperCAmelCase				:      Dict				=  datasets.map(
       snake_case__,   num_proc=data_args.preprocessing_num_workers,   remove_columns=datasets["train"].column_names				)

   # filter audio files that are too long
   _UpperCAmelCase				:      Tuple				=  vectorized_datasets.filter(
       lambda a_				: len(data["speech"]				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

   def normalize(a_:			List[str]				):
      return feature_extractor(batch["speech"],   sampling_rate=feature_extractor.sampling_rate				)

   # normalize and transform to `BatchFeatures`
   _UpperCAmelCase				:      int				=  vectorized_datasets.map(
       snake_case__,   batched=snake_case__,   num_proc=data_args.preprocessing_num_workers,   load_from_cache_file=not data_args.overwrite_cache,   remove_columns=vectorized_datasets["train"].column_names,   )

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   _UpperCAmelCase				:      Optional[Any]				=  WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path,   cache_dir=model_args.cache_dir,   gradient_checkpointing=training_args.gradient_checkpointing,   )

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
          " ``config.feat_extract_norm='layer'"				)

   _UpperCAmelCase				:      List[Any]				=  WavaVecaForPreTraining(snake_case__				)

   _UpperCAmelCase				:      Dict				=  DataCollatorForWavaVecaPretraining(model=snake_case__,   feature_extractor=snake_case__				)

   _UpperCAmelCase				:      int				=  WavaVecaPreTrainer(
       model=snake_case__,   data_collator=snake_case__,   args=snake_case__,   train_dataset=vectorized_datasets["train"],   eval_dataset=vectorized_datasets["validation"],   tokenizer=snake_case__,   max_gumbel_temp=model_args.max_gumbel_temperature,   min_gumbel_temp=model_args.min_gumbel_temperature,   gumbel_temp_decay=model_args.gumbel_temperature_decay,   )
   trainer.train()


if __name__ == "__main__":
     main()



import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__					:     Optional[int]       =     '▁'

SCREAMING_SNAKE_CASE__					:     Optional[Any]       =     get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
class 		_SCREAMING_SNAKE_CASE     (						A							,	unittest.TestCase     ):
     __SCREAMING_SNAKE_CASE     					=    BertGenerationTokenizer
     __SCREAMING_SNAKE_CASE     					=    False
     __SCREAMING_SNAKE_CASE     					=    True


     def        __snake_case(  self    ):
       super().setUp()

       _UpperCAmelCase		:							Optional[int]							=     BertGenerationTokenizer(A_							,					keep_accents=A_    )
       tokenizer.save_pretrained(self.tmpdirname    )


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Optional[int]							=     """<s>"""
       _UpperCAmelCase		:							Any							=     1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_    )							,					A_    )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_    )							,					A_    )


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Optional[int]							=     list(self.get_tokenizer().get_vocab().keys()    )

       self.assertEqual(vocab_keys[0]							,					"""<unk>"""    )
       self.assertEqual(vocab_keys[1]							,					"""<s>"""    )
       self.assertEqual(vocab_keys[-1]							,					"""<pad>"""    )
       self.assertEqual(len(A_    )							,					10_02    )


     def        __snake_case(  self    ):
       self.assertEqual(self.get_tokenizer().vocab_size							,					10_00    )


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Union[str, Any]							=     BertGenerationTokenizer(A_							,					keep_accents=A_    )

       _UpperCAmelCase		:							List[Any]							=     tokenizer.tokenize("""This is a test"""    )
       self.assertListEqual(A_							,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(A_    )							,					[2_85, 46, 10, 1_70, 3_82]							,					)

       _UpperCAmelCase		:							int							=     tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
       self.assertListEqual(
           A_							,					[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]							,					)
       _UpperCAmelCase		:							List[str]							=     tokenizer.convert_tokens_to_ids(A_    )
       self.assertListEqual(
           A_							,					[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,					)

       _UpperCAmelCase		:							Tuple							=     tokenizer.convert_ids_to_tokens(A_    )
       self.assertListEqual(
           A_							,					[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]							,					)


     @cached_property
     def        __snake_case(  self    ):
       return BertGenerationTokenizer.from_pretrained("""google/bert_for_seq_generation_L-24_bbc_encoder"""    )


     @slow
     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Optional[Any]							=     """Hello World!"""
       _UpperCAmelCase		:							str							=     [1_85_36, 22_60, 1_01]

       self.assertListEqual(A_							,					self.big_tokenizer.encode(A_    )    )


     @slow
     def        __snake_case(  self    ):
       _UpperCAmelCase		:							List[str]							=     (
           """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
           """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
       )
       _UpperCAmelCase		:							List[Any]							=     [
           8_71,
           4_19,
           3_58,
           9_46,
           9_91,
           25_21,
           4_52,
           3_58,
           13_57,
           3_87,
           77_51,
           35_36,
           1_12,
           9_85,
           4_56,
           1_26,
           8_65,
           9_38,
           54_00,
           57_34,
           4_58,
           13_68,
           4_67,
           7_86,
           24_62,
           52_46,
           11_59,
           6_33,
           8_65,
           45_19,
           4_57,
           5_82,
           8_52,
           25_57,
           4_27,
           9_16,
           5_08,
           4_05,
           3_43_24,
           4_97,
           3_91,
           4_08,
           1_13_42,
           12_44,
           3_85,
           1_00,
           9_38,
           9_85,
           4_56,
           5_74,
           3_62,
           1_25_97,
           32_00,
           31_29,
           11_72,
       ]

       self.assertListEqual(A_							,					self.big_tokenizer.encode(A_    )    )


     @require_torch
     @slow
     def        __snake_case(  self    ):
       import torch

       from transformers import BertGenerationConfig, BertGenerationEncoder

       # Build sequence
       _UpperCAmelCase		:							Optional[Any]							=     list(self.big_tokenizer.get_vocab().keys()    )[:10]
       _UpperCAmelCase		:							Tuple							=     """ """.join(A_    )
       _UpperCAmelCase		:							List[Any]							=     self.big_tokenizer.encode_plus(A_							,					return_tensors="""pt"""							,					return_token_type_ids=A_    )
       _UpperCAmelCase		:							Any							=     self.big_tokenizer.batch_encode_plus(
           [sequence + """ """ + sequence]							,					return_tensors="""pt"""							,					return_token_type_ids=A_    )

       _UpperCAmelCase		:							int							=     BertGenerationConfig()
       _UpperCAmelCase		:							Dict							=     BertGenerationEncoder(A_    )

       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
         model(**A_    )
         model(**A_    )




     @slow
     def        __snake_case(  self    ):
       # fmt: off
       _UpperCAmelCase		:							Optional[int]							=     {"""input_ids""": [[3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14], [4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=A_							,					model_name="""google/bert_for_seq_generation_L-24_bbc_encoder"""							,					revision="""c817d1fd1be2ffa69431227a1fe320544943d4db"""							,					)





