





'''simple docstring'''






import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
					from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
					lowerCAmelCase									=   True
					from torch.cuda.amp import autocast


lowerCAmelCase									=   logging.getLogger(__name__)
@dataclass
class 		lowerCamelCase	:
	snake_case_   = field(
	    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}							)
	snake_case_   = field(
	    default=_A   , metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}   , )
	snake_case_   = field(
	    default=_A   , metadata={"help": "Whether to freeze the feature extractor layers of the model."}							)
	snake_case_   = field(
	    default=_A   , metadata={"help": "Whether to log verbose messages or not."}   , )
	snake_case_   = field(
	    default=2.0   , metadata={"help": "Maximum temperature for gumbel softmax."}							)
	snake_case_   = field(
	    default=0.5   , metadata={"help": "Minimum temperature for gumbel softmax."}							)
	snake_case_   = field(
	    default=0.9_9_9_9_9_5   , metadata={"help": "Decay of gumbel temperature during training."}							)






def   __A    (				a_    :	ModelArguments   ,a_    :	TrainingArguments						):
		logging.basicConfig(
		    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,datefmt="%m/%d/%Y %H:%M:%S"   ,handlers=[logging.StreamHandler(sys.stdout						)]   ,)
		lowerCAmelCase  :     int       						= logging.WARNING
		if model_args.verbose_logging:
				lowerCAmelCase  :     Dict       						= logging.DEBUG
		elif trainer_utils.is_main_process(training_args.local_rank						):
				lowerCAmelCase  :     str       						= logging.INFO
		logger.setLevel(_a						)
@dataclass
class 		lowerCamelCase	:
	snake_case_   = field(
	    default=_A   , metadata={"help": "The name of the dataset to use (via the datasets library)."}							)
	snake_case_   = field(
	    default=_A   , metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}							)
	snake_case_   = field(
	    default="train"   , metadata={
	        "help": "The name of the training data set split to use (via the datasets library). Defaults to \'train\'"
	    }   , )
	snake_case_   = field(
	    default="validation"   , metadata={
	        "help": (
	            "The name of the validation data set split to use (via the datasets library). Defaults to \'validation\'"
	        )
	    }   , )
	snake_case_   = field(
	    default="file"   , metadata={"help": "Column in the dataset that contains speech file path. Defaults to \'file\'"}   , )
	snake_case_   = field(
	    default=_A   , metadata={"help": "Overwrite the cached preprocessed datasets or not."}							)
	snake_case_   = field(
	    default=1   , metadata={
	        "help": "The percentage of the train set used as validation set in case there\'s no validation split"
	    }   , )
	snake_case_   = field(
	    default=_A   , metadata={"help": "The number of processes to use for the preprocessing."}   , )
	snake_case_   = field(
	    default=20.0   , metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}							)






@dataclass
class 		lowerCamelCase	:
	snake_case_   = 42
	snake_case_   = 42
	snake_case_   = "longest"
	snake_case_   = None
	snake_case_   = None



	def __call__( self		,   a_						):
			# reformat list to dict and set to pytorch format
			lowerCAmelCase  :     Optional[int]       						= self.feature_extractor.pad(
			    lowerCamelCase_		,   max_length=self.max_length		,   padding=self.padding		,   pad_to_multiple_of=self.pad_to_multiple_of		,   return_tensors="pt"		,   )
			lowerCAmelCase  :     Optional[int]       						= self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]						)

			lowerCAmelCase  :     int       						= batch["input_values"].shape[0]

			# make sure that no loss is computed on padded inputs
			if batch["attention_mask"] is not None:
					# compute real output lengths according to convolution formula
					lowerCAmelCase  :     Any       						= self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1						)						).to(
					    torch.long						)

					lowerCAmelCase  :     int       						= torch.zeros(
					    (batch_size, mask_indices_seq_length)		,   dtype=torch.long		,   device=batch["input_values"].device						)

					# these two operations makes sure that all values
					# before the output lengths indices are attended to
					lowerCAmelCase  :     Optional[Any]       						= 1
					lowerCAmelCase  :     Optional[Any]       						= attention_mask.flip([-1]						).cumsum(-1						).flip([-1]						).bool()

			# sample randomly masked indices
			lowerCAmelCase  :     Union[str, Any]       						= _compute_mask_indices(
			    (batch_size, mask_indices_seq_length)		,   self.model.config.mask_time_prob		,   self.model.config.mask_time_length		,   attention_mask=lowerCamelCase_		,   min_masks=2		,   )

			return batch






class 		lowerCamelCase	(   _A							):



	def __init__( self		,   *a_		,   a_=1		,   a_=0		,   a_=1.0		,   **a_						):
			super().__init__(*lowerCamelCase_		,   **lowerCamelCase_						)
			lowerCAmelCase  :     Tuple       						= 0
			lowerCAmelCase  :     List[Any]       						= max_gumbel_temp
			lowerCAmelCase  :     Union[str, Any]       						= min_gumbel_temp
			lowerCAmelCase  :     Union[str, Any]       						= gumbel_temp_decay





	def 	_lowerCamelCase     ( self		,   a_		,   a_						):
			model.train()
			lowerCAmelCase  :     Union[str, Any]       						= self._prepare_inputs(lowerCamelCase_						)

			if self.use_amp:
					with autocast():
							lowerCAmelCase  :     List[Any]       						= self.compute_loss(lowerCamelCase_		,   lowerCamelCase_						)
			else:
					lowerCAmelCase  :     Any       						= self.compute_loss(lowerCamelCase_		,   lowerCamelCase_						)

			if self.args.n_gpu > 1 or self.deepspeed:
					if model.module.config.ctc_loss_reduction == "mean":
							lowerCAmelCase  :     Tuple       						= loss.mean()
					elif model.module.config.ctc_loss_reduction == "sum":
							lowerCAmelCase  :     Union[str, Any]       						= loss.sum() / (inputs["mask_time_indices"]).sum()
					else:
							raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''						)

			if self.args.gradient_accumulation_steps > 1:
					lowerCAmelCase  :     Tuple       						= loss / self.args.gradient_accumulation_steps

			if self.use_amp:
					self.scaler.scale(lowerCamelCase_						).backward()
			elif self.use_apex:
					with amp.scale_loss(lowerCamelCase_		,   self.optimizer						) as scaled_loss:
							scaled_loss.backward()
			elif self.deepspeed:
					self.deepspeed.backward(lowerCamelCase_						)
			else:
					loss.backward()

			self.num_update_step += 1
			# make sure gumbel softmax temperature is decayed
			if self.args.n_gpu > 1 or self.deepspeed:
					model.module.set_gumbel_temperature(
					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,   self.min_gumbel_temp						)						)
			else:
					model.set_gumbel_temperature(
					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,   self.min_gumbel_temp						)						)

			return loss.detach()






def   __A    (				):
		lowerCAmelCase  :     Dict       						= HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)						)

		lowerCAmelCase						,		lowerCAmelCase						,		lowerCAmelCase  :     Optional[int]       						= parser.parse_args_into_dataclasses()
		configure_logger(_a   ,_a						)

		# Downloading and loading a dataset from the hub.
		lowerCAmelCase  :     Optional[int]       						= load_dataset(data_args.dataset_name   ,data_args.dataset_config_name   ,cache_dir=model_args.cache_dir						)

		if "validation" not in datasets.keys():
				# make sure only "validation" and "train" keys remain"
				lowerCAmelCase  :     Dict       						= DatasetDict()
				lowerCAmelCase  :     Any       						= load_dataset(
				    data_args.dataset_name   ,data_args.dataset_config_name   ,split=f'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''   ,cache_dir=model_args.cache_dir   ,)
				lowerCAmelCase  :     Union[str, Any]       						= load_dataset(
				    data_args.dataset_name   ,data_args.dataset_config_name   ,split=f'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''   ,cache_dir=model_args.cache_dir   ,)
		else:
				# make sure only "validation" and "train" keys remain"
				lowerCAmelCase  :     List[str]       						= DatasetDict()
				lowerCAmelCase  :     Optional[int]       						= load_dataset(
				    data_args.dataset_name   ,data_args.dataset_config_name   ,split="validation"   ,cache_dir=model_args.cache_dir   ,)
				lowerCAmelCase  :     Union[str, Any]       						= load_dataset(
				    data_args.dataset_name   ,data_args.dataset_config_name   ,split=f'''{data_args.train_split_name}'''   ,cache_dir=model_args.cache_dir   ,)

		# only normalized-inputs-training is supported
		lowerCAmelCase  :     List[Any]       						= WavaVecaFeatureExtractor.from_pretrained(
		    model_args.model_name_or_path   ,cache_dir=model_args.cache_dir   ,do_normalize=_a						)

		def prepare_dataset(a_    :	Dict						):
				# check that all files have the correct sampling rate
				lowerCAmelCase						,		lowerCAmelCase  :     Optional[Any]       						= librosa.load(batch[data_args.speech_file_column]   ,sr=feature_extractor.sampling_rate						)
				return batch

		# load audio files into numpy arrays
		lowerCAmelCase  :     List[Any]       						= datasets.map(
		    _a   ,num_proc=data_args.preprocessing_num_workers   ,remove_columns=datasets["train"].column_names						)

		# filter audio files that are too long
		lowerCAmelCase  :     List[Any]       						= vectorized_datasets.filter(
		    lambda a_						: len(data["speech"]						) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate						)						)

		def normalize(a_    :	Optional[Any]						):
				return feature_extractor(batch["speech"]   ,sampling_rate=feature_extractor.sampling_rate						)

		# normalize and transform to `BatchFeatures`
		lowerCAmelCase  :     Tuple       						= vectorized_datasets.map(
		    _a   ,batched=_a   ,num_proc=data_args.preprocessing_num_workers   ,load_from_cache_file=not data_args.overwrite_cache   ,remove_columns=vectorized_datasets["train"].column_names   ,)

		# pretraining is only supported for "newer" stable layer norm architecture
		# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
		lowerCAmelCase  :     Any       						= WavaVecaConfig.from_pretrained(
		    model_args.model_name_or_path   ,cache_dir=model_args.cache_dir   ,gradient_checkpointing=training_args.gradient_checkpointing   ,)

		if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
				raise ValueError(
				    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
				    " ``config.feat_extract_norm='layer'"						)

		lowerCAmelCase  :     Optional[int]       						= WavaVecaForPreTraining(_a						)

		lowerCAmelCase  :     Optional[int]       						= DataCollatorForWavaVecaPretraining(model=_a   ,feature_extractor=_a						)

		lowerCAmelCase  :     Union[str, Any]       						= WavaVecaPreTrainer(
		    model=_a   ,data_collator=_a   ,args=_a   ,train_dataset=vectorized_datasets["train"]   ,eval_dataset=vectorized_datasets["validation"]   ,tokenizer=_a   ,max_gumbel_temp=model_args.max_gumbel_temperature   ,min_gumbel_temp=model_args.min_gumbel_temperature   ,gumbel_temp_decay=model_args.gumbel_temperature_decay   ,)
		trainer.train()


if __name__ == "__main__":
					main()






"""simple docstring"""


from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate





@dataclass(frozen=SCREAMING_SNAKE_CASE    )
class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   field(default='''summarization'''     ,   metadata={'''include_in_asdict_even_if_is_default''': True}    )
    _lowerCamelCase					=   Features({'''text''': Value('''string'''    )}    )
    _lowerCamelCase					=   Features({'''summary''': Value('''string'''    )}    )
    _lowerCamelCase					=   "text"
    _lowerCamelCase					=   "summary"




    @property
    def UpperCamelCase__		( self    )    ->      Dict[str, str]:
          return {self.text_column: "text", self.summary_column: "summary"}




