


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
			from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
			lowerCAmelCase       =							True
			from torch.cuda.amp import autocast


lowerCAmelCase       =							logging.getLogger(__name__)


@dataclass
class        _a							:
							_lowercase	:			str        = field(
							    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}			)
							_lowercase	:			Optional[str]        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}     ,	)
							_lowercase	:			Optional[bool]        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}			)
							_lowercase	:			Optional[bool]        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''Whether to log verbose messages or not.'''}     ,	)
							_lowercase	:			Optional[float]        = field(
							    default=2.0     ,	metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}			)
							_lowercase	:			Optional[float]        = field(
							    default=0.5     ,	metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}			)
							_lowercase	:			Optional[float]        = field(
							    default=0.999995     ,	metadata={'''help''': '''Decay of gumbel temperature during training.'''}			)
def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		logging.basicConfig(
		    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''						,				datefmt='''%m/%d/%Y %H:%M:%S'''						,				handlers=[logging.StreamHandler(sys.stdout     )]						,				)
		lowercase__            =      logging.WARNING
		if model_args.verbose_logging:
				lowercase__            =      logging.DEBUG
		elif trainer_utils.is_main_process(training_args.local_rank     ):
				lowercase__            =      logging.INFO
		logger.setLevel(SCREAMING_SNAKE_CASE     )


@dataclass
class        _a							:
							_lowercase	:			str        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}			)
							_lowercase	:			Optional[str]        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}			)
							_lowercase	:			Optional[str]        = field(
							    default='''train'''     ,	metadata={
							        '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
							    }     ,	)
							_lowercase	:			Optional[str]        = field(
							    default='''validation'''     ,	metadata={
							        '''help''': (
							            '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
							        )
							    }     ,	)
							_lowercase	:			Optional[str]        = field(
							    default='''file'''     ,	metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}     ,	)
							_lowercase	:			bool        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}			)
							_lowercase	:			Optional[int]        = field(
							    default=1     ,	metadata={
							        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
							    }     ,	)
							_lowercase	:			Optional[int]        = field(
							    default=UpperCamelCase__     ,	metadata={'''help''': '''The number of processes to use for the preprocessing.'''}     ,	)
							_lowercase	:			Optional[float]        = field(
							    default=20.0     ,	metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}			)


@dataclass
class        _a							:
							_lowercase	:			WavaVecaForPreTraining
							_lowercase	:			WavaVecaFeatureExtractor
							_lowercase	:			Union[bool, str]        = "longest"
							_lowercase	:			Optional[int]        = None
							_lowercase	:			Optional[int]        = None

							def __call__(							self:	Optional[Any]    ,						UpperCamelCase_:	List[Dict[str, Union[List[int], torch.Tensor]]]							)			->   Dict[str, torch.Tensor]:





									"""simple docstring"""







									lowercase__            =      self.feature_extractor.pad(
									    UpperCamelCase_    ,						max_length=self.max_length    ,						padding=self.padding    ,						pad_to_multiple_of=self.pad_to_multiple_of    ,						return_tensors='''pt'''    ,						)
									lowercase__            =      self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]							)

									lowercase__            =      batch['''input_values'''].shape[0]

									# make sure that no loss is computed on padded inputs
									if batch["attention_mask"] is not None:
											# compute real output lengths according to convolution formula
											lowercase__            =      self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1							)							).to(
											    torch.long							)

											lowercase__            =      torch.zeros(
											    (batch_size, mask_indices_seq_length)    ,						dtype=torch.long    ,						device=batch['''input_values'''].device							)

											# these two operations makes sure that all values
											# before the output lengths indices are attended to
											lowercase__            =      1
											lowercase__            =      attention_mask.flip([-1]							).cumsum(-1							).flip([-1]							).bool()

									# sample randomly masked indices
									lowercase__            =      _compute_mask_indices(
									    (batch_size, mask_indices_seq_length)    ,						self.model.config.mask_time_prob    ,						self.model.config.mask_time_length    ,						attention_mask=UpperCamelCase_    ,						min_masks=2    ,						)

									return batch


class        _a							(			UpperCamelCase__			):

							def __init__(							self:	Tuple    ,						*UpperCamelCase_:	Tuple    ,						UpperCamelCase_:	List[Any]=1    ,						UpperCamelCase_:	Optional[int]=0    ,						UpperCamelCase_:	int=1.0    ,						**UpperCamelCase_:	List[str]							)			->   Optional[int]:





									"""simple docstring"""







									super().__init__(*UpperCamelCase_    ,						**UpperCamelCase_							)
									lowercase__            =      0
									lowercase__            =      max_gumbel_temp
									lowercase__            =      min_gumbel_temp
									lowercase__            =      gumbel_temp_decay


							def       lowerCamelCase_ (							self:	List[str]    ,						UpperCamelCase_:	nn.Module    ,						UpperCamelCase_:	Dict[str, Union[torch.Tensor, Any]]							)			->   torch.Tensor:





									"""simple docstring"""







									model.train()
									lowercase__            =      self._prepare_inputs(UpperCamelCase_							)

									if self.use_amp:
											with autocast():
													lowercase__            =      self.compute_loss(UpperCamelCase_    ,						UpperCamelCase_							)
									else:
											lowercase__            =      self.compute_loss(UpperCamelCase_    ,						UpperCamelCase_							)

									if self.args.n_gpu > 1 or self.deepspeed:
											if model.module.config.ctc_loss_reduction == "mean":
													lowercase__            =      loss.mean()
											elif model.module.config.ctc_loss_reduction == "sum":
													lowercase__            =      loss.sum() / (inputs['''mask_time_indices''']).sum()
											else:
													raise ValueError(f'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'							)

									if self.args.gradient_accumulation_steps > 1:
											lowercase__            =      loss / self.args.gradient_accumulation_steps

									if self.use_amp:
											self.scaler.scale(UpperCamelCase_							).backward()
									elif self.use_apex:
											with amp.scale_loss(UpperCamelCase_    ,						self.optimizer							) as scaled_loss:
													scaled_loss.backward()
									elif self.deepspeed:
											self.deepspeed.backward(UpperCamelCase_							)
									else:
											loss.backward()

									self.num_update_step += 1
									# make sure gumbel softmax temperature is decayed
									if self.args.n_gpu > 1 or self.deepspeed:
											model.module.set_gumbel_temperature(
											    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,						self.min_gumbel_temp							)							)
									else:
											model.set_gumbel_temperature(
											    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,						self.min_gumbel_temp							)							)

									return loss.detach()
def 			_a (      ):


		"""simple docstring"""




		lowercase__            =      HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

		lowercase__     ,		lowercase__     ,		lowercase__            =      parser.parse_args_into_dataclasses()
		configure_logger(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     )

		# Downloading and loading a dataset from the hub.
		lowercase__            =      load_dataset(data_args.dataset_name						,				data_args.dataset_config_name						,				cache_dir=model_args.cache_dir     )

		if "validation" not in datasets.keys():
				# make sure only "validation" and "train" keys remain"
				lowercase__            =      DatasetDict()
				lowercase__            =      load_dataset(
				    data_args.dataset_name						,				data_args.dataset_config_name						,				split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'						,				cache_dir=model_args.cache_dir						,				)
				lowercase__            =      load_dataset(
				    data_args.dataset_name						,				data_args.dataset_config_name						,				split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'						,				cache_dir=model_args.cache_dir						,				)
		else:
				# make sure only "validation" and "train" keys remain"
				lowercase__            =      DatasetDict()
				lowercase__            =      load_dataset(
				    data_args.dataset_name						,				data_args.dataset_config_name						,				split='''validation'''						,				cache_dir=model_args.cache_dir						,				)
				lowercase__            =      load_dataset(
				    data_args.dataset_name						,				data_args.dataset_config_name						,				split=f'{data_args.train_split_name}'						,				cache_dir=model_args.cache_dir						,				)

		# only normalized-inputs-training is supported
		lowercase__            =      WavaVecaFeatureExtractor.from_pretrained(
		    model_args.model_name_or_path						,				cache_dir=model_args.cache_dir						,				do_normalize=SCREAMING_SNAKE_CASE     )

		def prepare_dataset(SCREAMING_SNAKE_CASE     ):
				# check that all files have the correct sampling rate
				lowercase__     ,		lowercase__            =      librosa.load(batch[data_args.speech_file_column]						,				sr=feature_extractor.sampling_rate     )
				return batch

		# load audio files into numpy arrays
		lowercase__            =      datasets.map(
		    SCREAMING_SNAKE_CASE						,				num_proc=data_args.preprocessing_num_workers						,				remove_columns=datasets['''train'''].column_names     )

		# filter audio files that are too long
		lowercase__            =      vectorized_datasets.filter(
		    lambda SCREAMING_SNAKE_CASE     : len(data['''speech''']     ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate     )     )

		def normalize(SCREAMING_SNAKE_CASE     ):
				return feature_extractor(batch['''speech''']						,				sampling_rate=feature_extractor.sampling_rate     )

		# normalize and transform to `BatchFeatures`
		lowercase__            =      vectorized_datasets.map(
		    SCREAMING_SNAKE_CASE						,				batched=SCREAMING_SNAKE_CASE						,				num_proc=data_args.preprocessing_num_workers						,				load_from_cache_file=not data_args.overwrite_cache						,				remove_columns=vectorized_datasets['''train'''].column_names						,				)

		# pretraining is only supported for "newer" stable layer norm architecture
		# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
		lowercase__            =      WavaVecaConfig.from_pretrained(
		    model_args.model_name_or_path						,				cache_dir=model_args.cache_dir						,				gradient_checkpointing=training_args.gradient_checkpointing						,				)

		if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
				raise ValueError(
				    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
				    ''' ``config.feat_extract_norm=\'layer\''''     )

		lowercase__            =      WavaVecaForPreTraining(SCREAMING_SNAKE_CASE     )

		lowercase__            =      DataCollatorForWavaVecaPretraining(model=SCREAMING_SNAKE_CASE						,				feature_extractor=SCREAMING_SNAKE_CASE     )

		lowercase__            =      WavaVecaPreTrainer(
		    model=SCREAMING_SNAKE_CASE						,				data_collator=SCREAMING_SNAKE_CASE						,				args=SCREAMING_SNAKE_CASE						,				train_dataset=vectorized_datasets['''train''']						,				eval_dataset=vectorized_datasets['''validation''']						,				tokenizer=SCREAMING_SNAKE_CASE						,				max_gumbel_temp=model_args.max_gumbel_temperature						,				min_gumbel_temp=model_args.min_gumbel_temperature						,				gumbel_temp_decay=model_args.gumbel_temperature_decay						,				)
		trainer.train()


if __name__ == "__main__":
			main()






def 			_a (      SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		if exponent == 1:
				return base
		if exponent % 2 == 0:
				lowercase__            =      _modexpt(SCREAMING_SNAKE_CASE						,				exponent // 2						,				SCREAMING_SNAKE_CASE     ) % modulo_value
				return (x * x) % modulo_value
		else:
				return (base * _modexpt(SCREAMING_SNAKE_CASE						,				exponent - 1						,				SCREAMING_SNAKE_CASE     )) % modulo_value
def 			_a (      SCREAMING_SNAKE_CASE = 17_77						,				SCREAMING_SNAKE_CASE = 18_55						,				SCREAMING_SNAKE_CASE = 8     ):


		"""simple docstring"""




		lowercase__            =      base
		for _ in range(1						,				SCREAMING_SNAKE_CASE     ):
				lowercase__            =      _modexpt(SCREAMING_SNAKE_CASE						,				SCREAMING_SNAKE_CASE						,				10**digits     )

		return result


if __name__ == "__main__":
			print(f"""{solution() = }""")


