import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
      from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
      A   :     Dict       =				True
      from torch.cuda.amp import autocast


A   :     Tuple       =				logging.getLogger(__name__)


@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(
      metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}     ,					)
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Whether to log verbose messages or not.'''}     ,					)
  lowerCamelCase__   =  field(
      default=2.0     ,					metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}    )
  lowerCamelCase__   =  field(
      default=0.5     ,					metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}    )
  lowerCamelCase__   =  field(
      default=0.999995     ,					metadata={'''help''': '''Decay of gumbel temperature during training.'''}    )



def    a__       (						__UpperCamelCase   ,					__UpperCamelCase ):
      logging.basicConfig(
          format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,					datefmt="%m/%d/%Y %H:%M:%S"   ,					handlers=[logging.StreamHandler(sys.stdout )]   ,					)
      SCREAMING_SNAKE_CASE_      =     logging.WARNING
      if model_args.verbose_logging:
            SCREAMING_SNAKE_CASE_      =     logging.DEBUG
      elif trainer_utils.is_main_process(training_args.local_rank ):
            SCREAMING_SNAKE_CASE_      =     logging.INFO
      logger.setLevel(__UpperCamelCase )


@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}    )
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}    )
  lowerCamelCase__   =  field(
      default='''train'''     ,					metadata={
          '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
      }     ,					)
  lowerCamelCase__   =  field(
      default='''validation'''     ,					metadata={
          '''help''': (
              '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
          )
      }     ,					)
  lowerCamelCase__   =  field(
      default='''file'''     ,					metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}     ,					)
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}    )
  lowerCamelCase__   =  field(
      default=1     ,					metadata={
          '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
      }     ,					)
  lowerCamelCase__   =  field(
      default=SCREAMING_SNAKE_CASE__     ,					metadata={'''help''': '''The number of processes to use for the preprocessing.'''}     ,					)
  lowerCamelCase__   =  field(
      default=20.0     ,					metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}    )



@dataclass
class      lowerCamelCase    :
  """simple docstring"""







  lowerCamelCase__   =  42
  lowerCamelCase__   =  42
  lowerCamelCase__   =  "longest"
  lowerCamelCase__   =  None
  lowerCamelCase__   =  None







  def __call__(							self       :				Tuple     ,   __magic_name__       :				List[Dict[str, Union[List[int], torch.Tensor]]]      )     ->		Dict[str, torch.Tensor]:
        # reformat list to dict and set to pytorch format
        SCREAMING_SNAKE_CASE_      =     self.feature_extractor.pad(
            __magic_name__     ,   max_length=self.max_length     ,   padding=self.padding     ,   pad_to_multiple_of=self.pad_to_multiple_of     ,   return_tensors="pt"     ,   )
        SCREAMING_SNAKE_CASE_      =     self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]      )

        SCREAMING_SNAKE_CASE_      =     batch["input_values"].shape[0]

        # make sure that no loss is computed on padded inputs
        if batch["attention_mask"] is not None:
              # compute real output lengths according to convolution formula
              SCREAMING_SNAKE_CASE_      =     self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1      )      ).to(
                  torch.long      )

              SCREAMING_SNAKE_CASE_      =     torch.zeros(
                  (batch_size, mask_indices_seq_length)     ,   dtype=torch.long     ,   device=batch["input_values"].device      )

              # these two operations makes sure that all values
              # before the output lengths indices are attended to
              SCREAMING_SNAKE_CASE_      =     1
              SCREAMING_SNAKE_CASE_      =     attention_mask.flip([-1]      ).cumsum(-1      ).flip([-1]      ).bool()

        # sample randomly masked indices
        SCREAMING_SNAKE_CASE_      =     _compute_mask_indices(
            (batch_size, mask_indices_seq_length)     ,   self.model.config.mask_time_prob     ,   self.model.config.mask_time_length     ,   attention_mask=__magic_name__     ,   min_masks=2     ,   )

        return batch



class      lowerCamelCase    (SCREAMING_SNAKE_CASE__    ):
  """simple docstring"""







  def __init__(							self       :				Optional[Any]     ,   *__magic_name__       :				Optional[Any]     ,   __magic_name__       :				Any=1     ,   __magic_name__       :				Tuple=0     ,   __magic_name__       :				Union[str, Any]=1.0     ,   **__magic_name__       :				Any      )     ->		Union[str, Any]:
        super().__init__(*__magic_name__     ,   **__magic_name__      )
        SCREAMING_SNAKE_CASE_      =     0
        SCREAMING_SNAKE_CASE_      =     max_gumbel_temp
        SCREAMING_SNAKE_CASE_      =     min_gumbel_temp
        SCREAMING_SNAKE_CASE_      =     gumbel_temp_decay







  def       __A   (							self       :				List[str]     ,   __magic_name__       :				nn.Module     ,   __magic_name__       :				Dict[str, Union[torch.Tensor, Any]]      )     ->		torch.Tensor:
        model.train()
        SCREAMING_SNAKE_CASE_      =     self._prepare_inputs(__magic_name__      )

        if self.use_amp:
              with autocast():
                    SCREAMING_SNAKE_CASE_      =     self.compute_loss(__magic_name__     ,   __magic_name__      )
        else:
              SCREAMING_SNAKE_CASE_      =     self.compute_loss(__magic_name__     ,   __magic_name__      )

        if self.args.n_gpu > 1 or self.deepspeed:
              if model.module.config.ctc_loss_reduction == "mean":
                    SCREAMING_SNAKE_CASE_      =     loss.mean()
              elif model.module.config.ctc_loss_reduction == "sum":
                    SCREAMING_SNAKE_CASE_      =     loss.sum() / (inputs["mask_time_indices"]).sum()
              else:
                    raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''      )

        if self.args.gradient_accumulation_steps > 1:
              SCREAMING_SNAKE_CASE_      =     loss / self.args.gradient_accumulation_steps

        if self.use_amp:
              self.scaler.scale(__magic_name__      ).backward()
        elif self.use_apex:
              with amp.scale_loss(__magic_name__     ,   self.optimizer      ) as scaled_loss:
                    scaled_loss.backward()
        elif self.deepspeed:
              self.deepspeed.backward(__magic_name__      )
        else:
              loss.backward()

        self.num_update_step += 1
        # make sure gumbel softmax temperature is decayed
        if self.args.n_gpu > 1 or self.deepspeed:
              model.module.set_gumbel_temperature(
                  max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,   self.min_gumbel_temp      )      )
        else:
              model.set_gumbel_temperature(
                  max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,   self.min_gumbel_temp      )      )

        return loss.detach()



def    a__       (						):
      # See all possible arguments in src/transformers/training_args.py
      # or by passing the --help flag to this script.
      # We now keep distinct sets of args, for a cleaner separation of concerns.

      SCREAMING_SNAKE_CASE_      =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

      SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_      =     parser.parse_args_into_dataclasses()
      configure_logger(__UpperCamelCase   ,					__UpperCamelCase )

      # Downloading and loading a dataset from the hub.
      SCREAMING_SNAKE_CASE_      =     load_dataset(data_args.dataset_name   ,					data_args.dataset_config_name   ,					cache_dir=model_args.cache_dir )

      if "validation" not in datasets.keys():
            # make sure only "validation" and "train" keys remain"
            SCREAMING_SNAKE_CASE_      =     DatasetDict()
            SCREAMING_SNAKE_CASE_      =     load_dataset(
                data_args.dataset_name   ,					data_args.dataset_config_name   ,					split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''   ,					cache_dir=model_args.cache_dir   ,					)
            SCREAMING_SNAKE_CASE_      =     load_dataset(
                data_args.dataset_name   ,					data_args.dataset_config_name   ,					split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''   ,					cache_dir=model_args.cache_dir   ,					)
      else:
            # make sure only "validation" and "train" keys remain"
            SCREAMING_SNAKE_CASE_      =     DatasetDict()
            SCREAMING_SNAKE_CASE_      =     load_dataset(
                data_args.dataset_name   ,					data_args.dataset_config_name   ,					split="validation"   ,					cache_dir=model_args.cache_dir   ,					)
            SCREAMING_SNAKE_CASE_      =     load_dataset(
                data_args.dataset_name   ,					data_args.dataset_config_name   ,					split=F'''{data_args.train_split_name}'''   ,					cache_dir=model_args.cache_dir   ,					)

      # only normalized-inputs-training is supported
      SCREAMING_SNAKE_CASE_      =     WavaVecaFeatureExtractor.from_pretrained(
          model_args.model_name_or_path   ,					cache_dir=model_args.cache_dir   ,					do_normalize=__UpperCamelCase )

      def prepare_dataset(__UpperCamelCase ):
            # check that all files have the correct sampling rate
            SCREAMING_SNAKE_CASE_   ,       SCREAMING_SNAKE_CASE_      =     librosa.load(batch[data_args.speech_file_column]   ,					sr=feature_extractor.sampling_rate )
            return batch

      # load audio files into numpy arrays
      SCREAMING_SNAKE_CASE_      =     datasets.map(
          __UpperCamelCase   ,					num_proc=data_args.preprocessing_num_workers   ,					remove_columns=datasets["train"].column_names )

      # filter audio files that are too long
      SCREAMING_SNAKE_CASE_      =     vectorized_datasets.filter(
          lambda __UpperCamelCase : len(data["speech"] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

      def normalize(__UpperCamelCase ):
            return feature_extractor(batch["speech"]   ,					sampling_rate=feature_extractor.sampling_rate )

      # normalize and transform to `BatchFeatures`
      SCREAMING_SNAKE_CASE_      =     vectorized_datasets.map(
          __UpperCamelCase   ,					batched=__UpperCamelCase   ,					num_proc=data_args.preprocessing_num_workers   ,					load_from_cache_file=not data_args.overwrite_cache   ,					remove_columns=vectorized_datasets["train"].column_names   ,					)

      # pretraining is only supported for "newer" stable layer norm architecture
      # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
      SCREAMING_SNAKE_CASE_      =     WavaVecaConfig.from_pretrained(
          model_args.model_name_or_path   ,					cache_dir=model_args.cache_dir   ,					gradient_checkpointing=training_args.gradient_checkpointing   ,					)

      if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
            raise ValueError(
                "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
                " ``config.feat_extract_norm='layer'" )

      SCREAMING_SNAKE_CASE_      =     WavaVecaForPreTraining(__UpperCamelCase )

      SCREAMING_SNAKE_CASE_      =     DataCollatorForWavaVecaPretraining(model=__UpperCamelCase   ,					feature_extractor=__UpperCamelCase )

      SCREAMING_SNAKE_CASE_      =     WavaVecaPreTrainer(
          model=__UpperCamelCase   ,					data_collator=__UpperCamelCase   ,					args=__UpperCamelCase   ,					train_dataset=vectorized_datasets["train"]   ,					eval_dataset=vectorized_datasets["validation"]   ,					tokenizer=__UpperCamelCase   ,					max_gumbel_temp=model_args.max_gumbel_temperature   ,					min_gumbel_temp=model_args.min_gumbel_temperature   ,					gumbel_temp_decay=model_args.gumbel_temperature_decay   ,					)
      trainer.train()


if __name__ == "__main__":
      main()



from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available


@dataclass
class      lowerCamelCase    (SCREAMING_SNAKE_CASE__    ):
  """simple docstring"""







  lowerCamelCase__   =  42
  lowerCamelCase__   =  42


if is_transformers_available() and is_torch_available():
      from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline


