


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
 from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
 SCREAMING_SNAKE_CASE__					:     List[Any]       =     True
 from torch.cuda.amp import autocast


SCREAMING_SNAKE_CASE__					:     Any       =     logging.getLogger(__name__)


@dataclass
class 		_SCREAMING_SNAKE_CASE     :
     __SCREAMING_SNAKE_CASE     					=    field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''Whether to log verbose messages or not.'''}							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default=2.0							,	metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=0.5							,	metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=0.99_9995							,	metadata={'''help''': '''Decay of gumbel temperature during training.'''}     )




def    a__    (		snake_case__					:							ModelArguments						,   snake_case__					:							TrainingArguments     ):
  logging.basicConfig(
      format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""						,   datefmt="""%m/%d/%Y %H:%M:%S"""						,   handlers=[logging.StreamHandler(sys.stdout     )]						,   )
  _UpperCAmelCase		:							List[Any]							=     logging.WARNING
  if model_args.verbose_logging:
    _UpperCAmelCase		:							Dict							=     logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank     ):
    _UpperCAmelCase		:							Optional[Any]							=     logging.INFO
  logger.setLevel(snake_case__     )


@dataclass
class 		_SCREAMING_SNAKE_CASE     :
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default='''train'''							,	metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         }							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default='''validation'''							,	metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         }							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default='''file'''							,	metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}     )
     __SCREAMING_SNAKE_CASE     					=    field(
         default=1							,	metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         }							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default=A							,	metadata={'''help''': '''The number of processes to use for the preprocessing.'''}							,	)
     __SCREAMING_SNAKE_CASE     					=    field(
         default=20.0							,	metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}     )


@dataclass
class 		_SCREAMING_SNAKE_CASE     :
     __SCREAMING_SNAKE_CASE     					=    42
     __SCREAMING_SNAKE_CASE     					=    42
     __SCREAMING_SNAKE_CASE     					=    "longest"
     __SCREAMING_SNAKE_CASE     					=    None
     __SCREAMING_SNAKE_CASE     					=    None


     def __call__(  self							,					A_    ):
       # reformat list to dict and set to pytorch format
       _UpperCAmelCase		:							Optional[int]							=     self.feature_extractor.pad(
           A_							,					max_length=self.max_length							,					padding=self.padding							,					pad_to_multiple_of=self.pad_to_multiple_of							,					return_tensors="""pt"""							,					)
       _UpperCAmelCase		:							str							=     self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]    )

       _UpperCAmelCase		:							Optional[int]							=     batch["""input_values"""].shape[0]

       # make sure that no loss is computed on padded inputs
       if batch["attention_mask"] is not None:
         # compute real output lengths according to convolution formula
         _UpperCAmelCase		:							Tuple							=     self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1    )    ).to(
             torch.long    )

         _UpperCAmelCase		:							Dict							=     torch.zeros(
             (batch_size, mask_indices_seq_length)							,					dtype=torch.long							,					device=batch["""input_values"""].device    )

         # these two operations makes sure that all values
         # before the output lengths indices are attended to
         _UpperCAmelCase		:							Tuple							=     1
         _UpperCAmelCase		:							int							=     attention_mask.flip([-1]    ).cumsum(-1    ).flip([-1]    ).bool()

       # sample randomly masked indices
       _UpperCAmelCase		:							Dict							=     _compute_mask_indices(
           (batch_size, mask_indices_seq_length)							,					self.model.config.mask_time_prob							,					self.model.config.mask_time_length							,					attention_mask=A_							,					min_masks=2							,					)

       return batch


class 		_SCREAMING_SNAKE_CASE     (						A     ):


     def __init__(  self							,					*A_							,					A_=1							,					A_=0							,					A_=1.0							,					**A_    ):
       super().__init__(*A_							,					**A_    )
       _UpperCAmelCase		:							Any							=     0
       _UpperCAmelCase		:							List[str]							=     max_gumbel_temp
       _UpperCAmelCase		:							List[Any]							=     min_gumbel_temp
       _UpperCAmelCase		:							str							=     gumbel_temp_decay




     def        __snake_case(  self							,					A_							,					A_    ):
       model.train()
       _UpperCAmelCase		:							int							=     self._prepare_inputs(A_    )

       if self.use_amp:
         with autocast():
           _UpperCAmelCase		:							Union[str, Any]							=     self.compute_loss(A_							,					A_    )
       else:
         _UpperCAmelCase		:							str							=     self.compute_loss(A_							,					A_    )

       if self.args.n_gpu > 1 or self.deepspeed:
         if model.module.config.ctc_loss_reduction == "mean":
           _UpperCAmelCase		:							int							=     loss.mean()
         elif model.module.config.ctc_loss_reduction == "sum":
           _UpperCAmelCase		:							Dict							=     loss.sum() / (inputs["""mask_time_indices"""]).sum()
         else:
           raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''    )

       if self.args.gradient_accumulation_steps > 1:
         _UpperCAmelCase		:							Tuple							=     loss / self.args.gradient_accumulation_steps

       if self.use_amp:
         self.scaler.scale(A_    ).backward()
       elif self.use_apex:
         with amp.scale_loss(A_							,					self.optimizer    ) as scaled_loss:
           scaled_loss.backward()
       elif self.deepspeed:
         self.deepspeed.backward(A_    )
       else:
         loss.backward()

       self.num_update_step += 1
       # make sure gumbel softmax temperature is decayed
       if self.args.n_gpu > 1 or self.deepspeed:
         model.module.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step							,					self.min_gumbel_temp    )    )
       else:
         model.set_gumbel_temperature(
             max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step							,					self.min_gumbel_temp    )    )

       return loss.detach()




def    a__    (		):
  # See all possible arguments in src/transformers/training_args.py
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.

  _UpperCAmelCase		:							int							=     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

  _UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase		:							Optional[int]							=     parser.parse_args_into_dataclasses()
  configure_logger(snake_case__						,   snake_case__     )

  # Downloading and loading a dataset from the hub.
  _UpperCAmelCase		:							Union[str, Any]							=     load_dataset(data_args.dataset_name						,   data_args.dataset_config_name						,   cache_dir=model_args.cache_dir     )

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    _UpperCAmelCase		:							Any							=     DatasetDict()
    _UpperCAmelCase		:							Any							=     load_dataset(
        data_args.dataset_name						,   data_args.dataset_config_name						,   split=f'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''						,   cache_dir=model_args.cache_dir						,   )
    _UpperCAmelCase		:							List[Any]							=     load_dataset(
        data_args.dataset_name						,   data_args.dataset_config_name						,   split=f'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''						,   cache_dir=model_args.cache_dir						,   )
  else:
    # make sure only "validation" and "train" keys remain"
    _UpperCAmelCase		:							Optional[Any]							=     DatasetDict()
    _UpperCAmelCase		:							Tuple							=     load_dataset(
        data_args.dataset_name						,   data_args.dataset_config_name						,   split="""validation"""						,   cache_dir=model_args.cache_dir						,   )
    _UpperCAmelCase		:							Union[str, Any]							=     load_dataset(
        data_args.dataset_name						,   data_args.dataset_config_name						,   split=f'''{data_args.train_split_name}'''						,   cache_dir=model_args.cache_dir						,   )

  # only normalized-inputs-training is supported
  _UpperCAmelCase		:							str							=     WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path						,   cache_dir=model_args.cache_dir						,   do_normalize=snake_case__     )

  def prepare_dataset(snake_case__					:							Union[str, Any]     ):
    # check that all files have the correct sampling rate
    _UpperCAmelCase,_UpperCAmelCase		:							Union[str, Any]							=     librosa.load(batch[data_args.speech_file_column]						,   sr=feature_extractor.sampling_rate     )
    return batch

  # load audio files into numpy arrays
  _UpperCAmelCase		:							Dict							=     datasets.map(
      snake_case__						,   num_proc=data_args.preprocessing_num_workers						,   remove_columns=datasets["""train"""].column_names     )

  # filter audio files that are too long
  _UpperCAmelCase		:							Tuple							=     vectorized_datasets.filter(
      lambda snake_case__     : len(data["""speech"""]     ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate     )     )

  def normalize(snake_case__					:							List[str]     ):
    return feature_extractor(batch["""speech"""]						,   sampling_rate=feature_extractor.sampling_rate     )

  # normalize and transform to `BatchFeatures`
  _UpperCAmelCase		:							int							=     vectorized_datasets.map(
      snake_case__						,   batched=snake_case__						,   num_proc=data_args.preprocessing_num_workers						,   load_from_cache_file=not data_args.overwrite_cache						,   remove_columns=vectorized_datasets["""train"""].column_names						,   )

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  _UpperCAmelCase		:							Optional[Any]							=     WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path						,   cache_dir=model_args.cache_dir						,   gradient_checkpointing=training_args.gradient_checkpointing						,   )

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
        """ ``config.feat_extract_norm='layer'"""     )

  _UpperCAmelCase		:							List[Any]							=     WavaVecaForPreTraining(snake_case__     )

  _UpperCAmelCase		:							Dict							=     DataCollatorForWavaVecaPretraining(model=snake_case__						,   feature_extractor=snake_case__     )

  _UpperCAmelCase		:							int							=     WavaVecaPreTrainer(
      model=snake_case__						,   data_collator=snake_case__						,   args=snake_case__						,   train_dataset=vectorized_datasets["""train"""]						,   eval_dataset=vectorized_datasets["""validation"""]						,   tokenizer=snake_case__						,   max_gumbel_temp=model_args.max_gumbel_temperature						,   min_gumbel_temp=model_args.min_gumbel_temperature						,   gumbel_temp_decay=model_args.gumbel_temperature_decay						,   )
  trainer.train()


if __name__ == "__main__":
 main()









from random import randint, random




def    a__    (		snake_case__					:							int						,   snake_case__					:							int						,   snake_case__					:							int						,   snake_case__					:							bool = False						,   snake_case__					:							bool = False						,   snake_case__					:							int = 5						,   ):
  _UpperCAmelCase		:							Optional[int]							=     [[-1] * number_of_cells]  # Create a highway without any car
  _UpperCAmelCase		:							Optional[Any]							=     0
  _UpperCAmelCase		:							Dict							=     max(snake_case__						,   0     )
  while i < number_of_cells:
    _UpperCAmelCase		:							int							=     (
        randint(0						,   snake_case__     ) if random_speed else initial_speed
    )  # Place the cars
    i += (
        randint(1						,   max_speed * 2     ) if random_frequency else frequency
    )  # Arbitrary number, may need tuning
  return highway




def    a__    (		snake_case__					:							list						,   snake_case__					:							int     ):
  _UpperCAmelCase		:							List[Any]							=     0
  _UpperCAmelCase		:							int							=     highway_now[car_index + 1 :]
  for cell in range(len(snake_case__     )     ):  # May need a better name for this
    if cells[cell] != -1:  # If the cell is not empty then
      return distance  # we have the distance we wanted
    distance += 1
  # Here if the car is near the end of the highway
  return distance + get_distance(snake_case__						,   -1     )




def    a__    (		snake_case__					:							list						,   snake_case__					:							float						,   snake_case__					:							int     ):
  _UpperCAmelCase		:							Optional[Any]							=     len(snake_case__     )
  # Beforce calculations, the highway is empty
  _UpperCAmelCase		:							Dict							=     [-1] * number_of_cells

  for car_index in range(snake_case__     ):
    if highway_now[car_index] != -1:
      # Add 1 to the current speed of the car and cap the speed
      _UpperCAmelCase		:							Dict							=     min(highway_now[car_index] + 1						,   snake_case__     )
      # Number of empty cell before the next car
      _UpperCAmelCase		:							List[str]							=     get_distance(snake_case__						,   snake_case__     ) - 1
      # We can't have the car causing an accident
      _UpperCAmelCase		:							List[str]							=     min(next_highway[car_index]						,   snake_case__     )
      if random() < probability:
        # Randomly, a driver will slow down
        _UpperCAmelCase		:							Dict							=     max(next_highway[car_index] - 1						,   0     )
  return next_highway




def    a__    (		snake_case__					:							list						,   snake_case__					:							int						,   snake_case__					:							float						,   snake_case__					:							int     ):
  _UpperCAmelCase		:							Union[str, Any]							=     len(highway[0]     )

  for i in range(snake_case__     ):
    _UpperCAmelCase		:							Tuple							=     update(highway[i]						,   snake_case__						,   snake_case__     )
    _UpperCAmelCase		:							int							=     [-1] * number_of_cells

    for car_index in range(snake_case__     ):
      _UpperCAmelCase		:							List[Any]							=     next_speeds_calculated[car_index]
      if speed != -1:
        # Change the position based on the speed (with % to create the loop)
        _UpperCAmelCase		:							Optional[Any]							=     (car_index + speed) % number_of_cells
        # Commit the change of position
        _UpperCAmelCase		:							Optional[Any]							=     speed
    highway.append(snake_case__     )

  return highway


if __name__ == "__main__":
 import doctest

 doctest.testmod()





