





'''simple docstring'''




import itertools
import os
import random
import tempfile
import unittest

import numpy as np

from transformers import TvltFeatureExtractor, is_datasets_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
			import torch

if is_datasets_available():
			from datasets import load_dataset

a__					:			Dict          =     random.Random()




def 						__lowerCamelCase			(	UpperCAmelCase_					,				UpperCAmelCase_=1.0					,				UpperCAmelCase_=None					,				UpperCAmelCase_=None							)   ->int:




	if rng is None:
		snake_case__	       =   global_rng

	snake_case__	       =   []
	for batch_idx in range(shape[0]							):
		values.append([]							)
		for _ in range(shape[1]							):
			values[-1].append(rng.random() * scale							)

	return values






class 				__snake_case     (					unittest.TestCase							):





	def __init__(		self      ,       UpperCamelCase_      ,       UpperCamelCase_=7      ,       UpperCamelCase_=400      ,       UpperCamelCase_=2000      ,       UpperCamelCase_=2048      ,       UpperCamelCase_=128      ,       UpperCamelCase_=1      ,       UpperCamelCase_=512      ,       UpperCamelCase_=30      ,       UpperCamelCase_=4_4100      ,       )      ->		int:
		snake_case__	       =   parent
		snake_case__	       =   batch_size
		snake_case__	       =   min_seq_length
		snake_case__	       =   max_seq_length
		snake_case__	       =   (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
		snake_case__	       =   spectrogram_length
		snake_case__	       =   feature_size
		snake_case__	       =   num_audio_channels
		snake_case__	       =   hop_length
		snake_case__	       =   chunk_length
		snake_case__	       =   sampling_rate





	def   _snake_case   (		self		)      ->		Dict:
		return {
		    "spectrogram_length": self.spectrogram_length,
		    "feature_size": self.feature_size,
		    "num_audio_channels": self.num_audio_channels,
		    "hop_length": self.hop_length,
		    "chunk_length": self.chunk_length,
		    "sampling_rate": self.sampling_rate,
		}





	def   _snake_case   (		self      ,       UpperCamelCase_=False      ,       UpperCamelCase_=False		)      ->		Optional[int]:
		def _flatten(UpperCamelCase_		):
			return list(itertools.chain(*__a		)		)

		if equal_length:
			snake_case__	       =   [floats_list((self.max_seq_length, self.feature_size)		) for _ in range(self.batch_size		)]
		else:
			# make sure that inputs increase in size
			snake_case__	       =   [
			    floats_list((x, self.feature_size)		)
			    for x in range(self.min_seq_length      ,       self.max_seq_length      ,       self.seq_length_diff		)
			]
		if numpify:
			snake_case__	       =   [np.asarray(__a		) for x in speech_inputs]
		return speech_inputs






@require_torch
@require_torchaudio
class 				__snake_case     (					lowerCAmelCase__       ,       unittest.TestCase							):
	__lowerCAmelCase       						=  TvltFeatureExtractor





	def   _snake_case   (		self		)      ->		Any:
		snake_case__	       =   TvltFeatureExtractionTester(self		)





	def   _snake_case   (		self		)      ->		Any:
		snake_case__	       =   self.feature_extraction_class(**self.feat_extract_dict		)
		self.assertTrue(hasattr(__a      ,       'spectrogram_length'		)		)
		self.assertTrue(hasattr(__a      ,       'feature_size'		)		)
		self.assertTrue(hasattr(__a      ,       'num_audio_channels'		)		)
		self.assertTrue(hasattr(__a      ,       'hop_length'		)		)
		self.assertTrue(hasattr(__a      ,       'chunk_length'		)		)
		self.assertTrue(hasattr(__a      ,       'sampling_rate'		)		)





	def   _snake_case   (		self		)      ->		List[str]:
		snake_case__	       =   self.feature_extraction_class(**self.feat_extract_dict		)

		with tempfile.TemporaryDirectory() as tmpdirname:
			snake_case__	       =   feat_extract_first.save_pretrained(__a		)[0]
			check_json_file_has_correct_format(__a		)
			snake_case__	       =   self.feature_extraction_class.from_pretrained(__a		)

		snake_case__	       =   feat_extract_first.to_dict()
		snake_case__	       =   feat_extract_second.to_dict()
		snake_case__	       =   dict_first.pop('mel_filters'		)
		snake_case__	       =   dict_second.pop('mel_filters'		)
		self.assertTrue(np.allclose(__a      ,       __a		)		)
		self.assertEqual(__a      ,       __a		)





	def   _snake_case   (		self		)      ->		List[Any]:
		snake_case__	       =   self.feature_extraction_class(**self.feat_extract_dict		)

		with tempfile.TemporaryDirectory() as tmpdirname:
			snake_case__	       =   os.path.join(__a      ,       'feat_extract.json'		)
			feat_extract_first.to_json_file(__a		)
			snake_case__	       =   self.feature_extraction_class.from_json_file(__a		)

		snake_case__	       =   feat_extract_first.to_dict()
		snake_case__	       =   feat_extract_second.to_dict()
		snake_case__	       =   dict_first.pop('mel_filters'		)
		snake_case__	       =   dict_second.pop('mel_filters'		)
		self.assertTrue(np.allclose(__a      ,       __a		)		)
		self.assertEqual(__a      ,       __a		)





	def   _snake_case   (		self		)      ->		Optional[int]:
		# Initialize feature_extractor
		snake_case__	       =   self.feature_extraction_class(**self.feat_extract_dict		)

		# create three inputs of length 800, 1000, and 1200
		snake_case__	       =   [floats_list((1, x)		)[0] for x in range(800      ,       1400      ,       200		)]
		snake_case__	       =   [np.asarray(__a		) for speech_input in speech_inputs]

		# Test not batched input
		snake_case__	       =   feature_extractor(np_speech_inputs[0]      ,       return_tensors='np'      ,       sampling_rate=4_4100		).audio_values

		self.assertTrue(encoded_audios.ndim == 4		)
		self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
		self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
		self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

		# Test batched
		snake_case__	       =   feature_extractor(__a      ,       return_tensors='np'      ,       sampling_rate=4_4100		).audio_values

		self.assertTrue(encoded_audios.ndim == 4		)
		self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
		self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
		self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

		# Test audio masking
		snake_case__	       =   feature_extractor(
		    __a      ,       return_tensors='np'      ,       sampling_rate=4_4100      ,       mask_audio=__a		).audio_values

		self.assertTrue(encoded_audios.ndim == 4		)
		self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
		self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
		self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)

		# Test 2-D numpy arrays are batched.
		snake_case__	       =   [floats_list((1, x)		)[0] for x in (800, 800, 800)]
		snake_case__	       =   np.asarray(__a		)
		snake_case__	       =   feature_extractor(__a      ,       return_tensors='np'      ,       sampling_rate=4_4100		).audio_values
		self.assertTrue(encoded_audios.ndim == 4		)
		self.assertTrue(encoded_audios.shape[-1] == feature_extractor.feature_size		)
		self.assertTrue(encoded_audios.shape[-2] <= feature_extractor.spectrogram_length		)
		self.assertTrue(encoded_audios.shape[-3] == feature_extractor.num_channels		)





	def   _snake_case   (		self      ,       UpperCamelCase_		)      ->		Tuple:
		snake_case__	       =   load_dataset('hf-internal-testing/librispeech_asr_dummy'      ,       'clean'      ,       split='validation'		)
		# automatic decoding with librispeech
		snake_case__	       =   ds.sort('id'		).select(range(__a		)		)[:num_samples]['audio']

		return [x["array"] for x in speech_samples]





	def   _snake_case   (		self		)      ->		int:
		snake_case__	       =   self._load_datasamples(1		)
		snake_case__	       =   TvltFeatureExtractor()
		snake_case__	       =   feature_extractor(__a      ,       return_tensors='pt'		).audio_values

		self.assertEquals(audio_values.shape      ,       (1, 1, 192, 128)		)

		snake_case__	       =   torch.tensor([[-0.3_0_3_2, -0.2_7_0_8], [-0.4_4_3_4, -0.4_0_0_7]]		)
		self.assertTrue(torch.allclose(audio_values[0, 0, :2, :2]      ,       __a      ,       atol=1E-4		)		)







"""simple docstring"""


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
	from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
	A	:  List[Any]            =     True
	from torch.cuda.amp import autocast


A	:  Any            =     logging.getLogger(__name__)
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       str							 =field(
							    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,)
							__UpperCAmelCase		:       Optional[bool]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}						)
							__UpperCAmelCase		:       Optional[bool]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Whether to log verbose messages or not."""}			,)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=2.0			,metadata={"""help""": """Maximum temperature for gumbel softmax."""}						)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=0.5			,metadata={"""help""": """Minimum temperature for gumbel softmax."""}						)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=0.999_995			,metadata={"""help""": """Decay of gumbel temperature during training."""}						)


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			logging.basicConfig(
			    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,					datefmt="%m/%d/%Y %H:%M:%S"       ,					handlers=[logging.StreamHandler(sys.stdout )]       ,					)
			__lowerCAmelCase							       =     logging.WARNING
			if model_args.verbose_logging:
						__lowerCAmelCase							       =     logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank ):
						__lowerCAmelCase							       =     logging.INFO
			logger.setLevel(_UpperCamelCase )
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       str							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""train"""			,metadata={
							        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
							    }			,)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""validation"""			,metadata={
							        """help""": (
							            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
							        )
							    }			,)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""file"""			,metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}			,)
							__UpperCAmelCase		:       bool							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}						)
							__UpperCAmelCase		:       Optional[int]							 =field(
							    default=1			,metadata={
							        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
							    }			,)
							__UpperCAmelCase		:       Optional[int]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The number of processes to use for the preprocessing."""}			,)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=20.0			,metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}						)
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       WavaVecaForPreTraining
							__UpperCAmelCase		:       WavaVecaFeatureExtractor
							__UpperCAmelCase		:       Union[bool, str]							 ="longest"
							__UpperCAmelCase		:       Optional[int]							 =None
							__UpperCAmelCase		:       Optional[int]							 =None

							def __call__(	self       ,  __a      ):
										# reformat list to dict and set to pytorch format
										__lowerCAmelCase							       =     self.feature_extractor.pad(
										    __a       ,  max_length=self.max_length       ,  padding=self.padding       ,  pad_to_multiple_of=self.pad_to_multiple_of       ,  return_tensors="pt"       ,  )
										__lowerCAmelCase							       =     self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]      )

										__lowerCAmelCase							       =     batch["input_values"].shape[0]

										# make sure that no loss is computed on padded inputs
										if batch["attention_mask"] is not None:
													# compute real output lengths according to convolution formula
													__lowerCAmelCase							       =     self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1      )      ).to(
													    torch.long      )

													__lowerCAmelCase							       =     torch.zeros(
													    (batch_size, mask_indices_seq_length)       ,  dtype=torch.long       ,  device=batch["input_values"].device      )

													# these two operations makes sure that all values
													# before the output lengths indices are attended to
													__lowerCAmelCase							       =     1
													__lowerCAmelCase							       =     attention_mask.flip([-1]      ).cumsum(-1      ).flip([-1]      ).bool()

										# sample randomly masked indices
										__lowerCAmelCase							       =     _compute_mask_indices(
										    (batch_size, mask_indices_seq_length)       ,  self.model.config.mask_time_prob       ,  self.model.config.mask_time_length       ,  attention_mask=__a       ,  min_masks=2       ,  )

										return batch
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  *__a       ,  __a=1       ,  __a=0       ,  __a=1.0       ,  **__a      ):
										super().__init__(*__a       ,  **__a      )
										__lowerCAmelCase							       =     0
										__lowerCAmelCase							       =     max_gumbel_temp
										__lowerCAmelCase							       =     min_gumbel_temp
										__lowerCAmelCase							       =     gumbel_temp_decay



							def      snake_case							(	self       ,  __a       ,  __a      ):
										model.train()
										__lowerCAmelCase							       =     self._prepare_inputs(__a      )

										if self.use_amp:
													with autocast():
																__lowerCAmelCase							       =     self.compute_loss(__a       ,  __a      )
										else:
													__lowerCAmelCase							       =     self.compute_loss(__a       ,  __a      )

										if self.args.n_gpu > 1 or self.deepspeed:
													if model.module.config.ctc_loss_reduction == "mean":
																__lowerCAmelCase							       =     loss.mean()
													elif model.module.config.ctc_loss_reduction == "sum":
																__lowerCAmelCase							       =     loss.sum() / (inputs["mask_time_indices"]).sum()
													else:
																raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"      )

										if self.args.gradient_accumulation_steps > 1:
													__lowerCAmelCase							       =     loss / self.args.gradient_accumulation_steps

										if self.use_amp:
													self.scaler.scale(__a      ).backward()
										elif self.use_apex:
													with amp.scale_loss(__a       ,  self.optimizer      ) as scaled_loss:
																scaled_loss.backward()
										elif self.deepspeed:
													self.deepspeed.backward(__a      )
										else:
													loss.backward()

										self.num_update_step += 1
										# make sure gumbel softmax temperature is decayed
										if self.args.n_gpu > 1 or self.deepspeed:
													model.module.set_gumbel_temperature(
													    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,  self.min_gumbel_temp      )      )
										else:
													model.set_gumbel_temperature(
													    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,  self.min_gumbel_temp      )      )

										return loss.detach()


def _lowerCamelCase      (						):

			'''simple docstring'''




			__lowerCAmelCase							       =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

			__lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase							       =     parser.parse_args_into_dataclasses()
			configure_logger(_UpperCamelCase       ,					_UpperCamelCase )

			# Downloading and loading a dataset from the hub.
			__lowerCAmelCase							       =     load_dataset(data_args.dataset_name       ,					data_args.dataset_config_name       ,					cache_dir=model_args.cache_dir )

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						__lowerCAmelCase							       =     DatasetDict()
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"       ,					cache_dir=model_args.cache_dir       ,					)
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"       ,					cache_dir=model_args.cache_dir       ,					)
			else:
						# make sure only "validation" and "train" keys remain"
						__lowerCAmelCase							       =     DatasetDict()
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split="validation"       ,					cache_dir=model_args.cache_dir       ,					)
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}"       ,					cache_dir=model_args.cache_dir       ,					)

			# only normalized-inputs-training is supported
			__lowerCAmelCase							       =     WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path       ,					cache_dir=model_args.cache_dir       ,					do_normalize=_UpperCamelCase )

			def prepare_dataset(_UpperCamelCase ):
						# check that all files have the correct sampling rate
						__lowerCAmelCase       ,      __lowerCAmelCase							       =     librosa.load(batch[data_args.speech_file_column]       ,					sr=feature_extractor.sampling_rate )
						return batch

			# load audio files into numpy arrays
			__lowerCAmelCase							       =     datasets.map(
			    _UpperCamelCase       ,					num_proc=data_args.preprocessing_num_workers       ,					remove_columns=datasets["train"].column_names )

			# filter audio files that are too long
			__lowerCAmelCase							       =     vectorized_datasets.filter(
			    lambda _UpperCamelCase : len(data["speech"] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

			def normalize(_UpperCamelCase ):
						return feature_extractor(batch["speech"]       ,					sampling_rate=feature_extractor.sampling_rate )

			# normalize and transform to `BatchFeatures`
			__lowerCAmelCase							       =     vectorized_datasets.map(
			    _UpperCamelCase       ,					batched=_UpperCamelCase       ,					num_proc=data_args.preprocessing_num_workers       ,					load_from_cache_file=not data_args.overwrite_cache       ,					remove_columns=vectorized_datasets["train"].column_names       ,					)

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			__lowerCAmelCase							       =     WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path       ,					cache_dir=model_args.cache_dir       ,					gradient_checkpointing=training_args.gradient_checkpointing       ,					)

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
						    " ``config.feat_extract_norm='layer'" )

			__lowerCAmelCase							       =     WavaVecaForPreTraining(_UpperCamelCase )

			__lowerCAmelCase							       =     DataCollatorForWavaVecaPretraining(model=_UpperCamelCase       ,					feature_extractor=_UpperCamelCase )

			__lowerCAmelCase							       =     WavaVecaPreTrainer(
			    model=_UpperCamelCase       ,					data_collator=_UpperCamelCase       ,					args=_UpperCamelCase       ,					train_dataset=vectorized_datasets["train"]       ,					eval_dataset=vectorized_datasets["validation"]       ,					tokenizer=_UpperCamelCase       ,					max_gumbel_temp=model_args.max_gumbel_temperature       ,					min_gumbel_temp=model_args.min_gumbel_temperature       ,					gumbel_temp_decay=model_args.gumbel_temperature_decay       ,					)
			trainer.train()


if __name__ == "__main__":
	main()




