


"""simple docstring"""


import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_flax,
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import is_flax_available, is_torch_available, is_vision_available

from ...test_modeling_flax_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_flax_bert import FlaxBertModelTester
from ..clip.test_modeling_flax_clip import FlaxCLIPVisionModelTester
from ..vit.test_modeling_flax_vit import FlaxViTModelTester


if is_flax_available():
	from transformers import (
	    FlaxBertModel,
	    FlaxCLIPVisionModel,
	    FlaxVisionTextDualEncoderModel,
	    FlaxViTModel,
	    VisionTextDualEncoderConfig,
	    VisionTextDualEncoderProcessor,
	)
	from transformers.modeling_flax_pytorch_utils import (
	    convert_pytorch_state_dict_to_flax,
	    load_flax_weights_in_pytorch_model,
	)


if is_torch_available():
	import torch

	from transformers import VisionTextDualEncoderModel

if is_vision_available():
	from PIL import Image


def _lowerCamelCase      (						_UpperCamelCase ):

			'''simple docstring'''




			if isinstance(_UpperCamelCase       ,					collections.abc.Iterable ):
						return x
			return (x, x)
@require_flax
class     _UpperCamelCase    :
							'''simple docstring'''







							def      snake_case							(	self       ,  __a       ,  __a      ):
										pass

							def      snake_case							(	self      ):
										pass

							def      snake_case							(	self      ):
										pass

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     np.abs((a - b)      ).max()
										self.assertLessEqual(__a       ,  __a       ,  f"Difference between torch and flax is {diff} (>= {tol})."      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a       ,  __a       ,  __a=None       ,  **__a      ):
										__lowerCAmelCase							       =     VisionTextDualEncoderConfig.from_vision_text_configs(__a       ,  __a      )

										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel(__a      )

										__lowerCAmelCase							       =     model(input_ids=__a       ,  pixel_values=__a       ,  attention_mask=__a      )

										self.assertEqual(output["text_embeds"].shape       ,  (input_ids.shape[0], config.projection_dim)      )
										self.assertEqual(output["image_embeds"].shape       ,  (pixel_values.shape[0], config.projection_dim)      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a       ,  __a       ,  __a=None       ,  **__a      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     self.get_vision_text_model(__a       ,  __a      )
										__lowerCAmelCase							       =     {"vision_model": vision_model, "text_model": text_model}
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__a      )

										__lowerCAmelCase							       =     model(input_ids=__a       ,  pixel_values=__a       ,  attention_mask=__a      )

										self.assertEqual(output["text_embeds"].shape       ,  (input_ids.shape[0], model.config.projection_dim)      )
										self.assertEqual(output["image_embeds"].shape       ,  (pixel_values.shape[0], model.config.projection_dim)      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a       ,  __a       ,  __a=None       ,  **__a      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     self.get_vision_text_model(__a       ,  __a      )
										__lowerCAmelCase							       =     {"vision_model": vision_model, "text_model": text_model}
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__a      )

										__lowerCAmelCase							       =     model(input_ids=__a       ,  pixel_values=__a       ,  attention_mask=__a      )
										__lowerCAmelCase							       =     output[0]

										with tempfile.TemporaryDirectory() as tmpdirname:
													model.save_pretrained(__a      )
													__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_pretrained(__a      )

													__lowerCAmelCase							       =     model(input_ids=__a       ,  pixel_values=__a       ,  attention_mask=__a      )
													__lowerCAmelCase							       =     after_output[0]
													__lowerCAmelCase							       =     np.amax(np.abs(out_a - out_a      )      )
													self.assertLessEqual(__a       ,  1e-3      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a       ,  __a       ,  __a=None       ,  **__a      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     self.get_vision_text_model(__a       ,  __a      )
										__lowerCAmelCase							       =     {"vision_model": vision_model, "text_model": text_model}
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(**__a      )

										__lowerCAmelCase							       =     model(
										    input_ids=__a       ,  pixel_values=__a       ,  attention_mask=__a       ,  output_attentions=__a      )

										__lowerCAmelCase							       =     output.vision_model_output.attentions
										self.assertEqual(len(__a      )       ,  vision_config.num_hidden_layers      )

										# in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
										__lowerCAmelCase							       =     to_atuple(vision_model.config.image_size      )
										__lowerCAmelCase							       =     to_atuple(vision_model.config.patch_size      )
										__lowerCAmelCase							       =     (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
										__lowerCAmelCase							       =     num_patches + 1
										self.assertEqual(vision_attentions[0].shape[-3:]       ,  (vision_config.num_attention_heads, seq_len, seq_len)      )

										__lowerCAmelCase							       =     output.text_model_output.attentions
										self.assertEqual(len(__a      )       ,  text_config.num_hidden_layers      )

										self.assertEqual(
										    text_attentions[0].shape[-3:]       ,  (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])       ,  )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a      ):
										pt_model.to(__a      )
										pt_model.eval()

										# prepare inputs
										__lowerCAmelCase							       =     inputs_dict
										__lowerCAmelCase							       =     {k: torch.tensor(v.tolist()      ) for k, v in flax_inputs.items()}

										with torch.no_grad():
													__lowerCAmelCase							       =     pt_model(**__a      ).to_tuple()

										__lowerCAmelCase							       =     fx_model(**__a      ).to_tuple()
										self.assertEqual(len(__a      )       ,  len(__a      )       ,  "Output lengths differ between Flax and PyTorch"      )
										for fx_output, pt_output in zip(fx_outputs[:4]       ,  pt_outputs[:4]      ):
													self.assert_almost_equals(__a       ,  pt_output.numpy()       ,  4e-2      )

										# PT -> Flax
										with tempfile.TemporaryDirectory() as tmpdirname:
													pt_model.save_pretrained(__a      )
													__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_pretrained(__a       ,  from_pt=__a      )

										__lowerCAmelCase							       =     fx_model_loaded(**__a      ).to_tuple()
										self.assertEqual(len(__a      )       ,  len(__a      )       ,  "Output lengths differ between Flax and PyTorch"      )
										for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4]       ,  pt_outputs[:4]      ):
													self.assert_almost_equals(__a       ,  pt_output.numpy()       ,  4e-2      )

										# Flax -> PT
										with tempfile.TemporaryDirectory() as tmpdirname:
													fx_model.save_pretrained(__a      )
													__lowerCAmelCase							       =     VisionTextDualEncoderModel.from_pretrained(__a       ,  from_flax=__a      )

										pt_model_loaded.to(__a      )
										pt_model_loaded.eval()

										with torch.no_grad():
													__lowerCAmelCase							       =     pt_model_loaded(**__a      ).to_tuple()

										self.assertEqual(len(__a      )       ,  len(__a      )       ,  "Output lengths differ between Flax and PyTorch"      )
										for fx_output, pt_output_loaded in zip(fx_outputs[:4]       ,  pt_outputs_loaded[:4]      ):
													self.assert_almost_equals(__a       ,  pt_output_loaded.numpy()       ,  4e-2      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     VisionTextDualEncoderConfig.from_vision_text_configs(__a       ,  __a      )

										__lowerCAmelCase							       =     VisionTextDualEncoderModel(__a      )
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel(__a      )

										__lowerCAmelCase							       =     convert_pytorch_state_dict_to_flax(pt_model.state_dict()       ,  __a      )
										__lowerCAmelCase							       =     fx_state

										self.check_pt_flax_equivalence(__a       ,  __a       ,  __a      )

							def      snake_case							(	self       ,  __a       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     VisionTextDualEncoderConfig.from_vision_text_configs(__a       ,  __a      )

										__lowerCAmelCase							       =     VisionTextDualEncoderModel(__a      )
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel(__a      )

										__lowerCAmelCase							       =     load_flax_weights_in_pytorch_model(__a       ,  fx_model.params      )

										self.check_pt_flax_equivalence(__a       ,  __a       ,  __a      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     self.prepare_config_and_inputs()
										self.check_model_from_pretrained_configs(**__a      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     self.prepare_config_and_inputs()
										self.check_vision_text_dual_encoder_from_pretrained(**__a      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     self.prepare_config_and_inputs()
										self.check_save_load(**__a      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     self.prepare_config_and_inputs()
										self.check_vision_text_output_attention(**__a      )

							@is_pt_flax_cross_test
							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     self.prepare_config_and_inputs()
										__lowerCAmelCase							       =     config_inputs_dict.pop("vision_config"      )
										__lowerCAmelCase							       =     config_inputs_dict.pop("text_config"      )

										__lowerCAmelCase							       =     config_inputs_dict

										self.check_equivalence_pt_to_flax(__a       ,  __a       ,  __a      )
										self.check_equivalence_flax_to_pt(__a       ,  __a       ,  __a      )



							@slow
							def      snake_case							(	self      ):
										__lowerCAmelCase       ,      __lowerCAmelCase							       =     self.get_pretrained_model_and_inputs()

										__lowerCAmelCase							       =     model_a(**__a      )
										__lowerCAmelCase							       =     outputs[0]

										with tempfile.TemporaryDirectory() as tmp_dirname:
													model_a.save_pretrained(__a      )
													__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_pretrained(__a      )

													__lowerCAmelCase							       =     model_a(**__a      )
													__lowerCAmelCase							       =     after_outputs[0]
													__lowerCAmelCase							       =     np.amax(np.abs(out_a - out_a      )      )
													self.assertLessEqual(__a       ,  1e-5      )
@require_flax
class     _UpperCamelCase    (  lowerCAmelCase__			,unittest.TestCase						):
							'''simple docstring'''







							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
										    "hf-internal-testing/tiny-random-vit"       ,  "hf-internal-testing/tiny-bert"       ,  vision_from_pt=__a       ,  text_from_pt=__a       ,  )
										__lowerCAmelCase							       =     13
										__lowerCAmelCase							       =     floats_tensor(
										    [
										        batch_size,
										        model.config.vision_config.num_channels,
										        model.config.vision_config.image_size,
										        model.config.vision_config.image_size,
										    ]      )
										__lowerCAmelCase							       =     ids_tensor([batch_size, 4]       ,  model.config.text_config.vocab_size      )
										__lowerCAmelCase							       =     random_attention_mask([batch_size, 4]      )
										__lowerCAmelCase							       =     {"pixel_values": pixel_values, "input_ids": input_ids, "attention_mask": attention_mask}

										return model, inputs

							def      snake_case							(	self       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     FlaxViTModel(__a      )
										__lowerCAmelCase							       =     FlaxBertModel(__a      )
										return vision_model, text_model



							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     FlaxViTModelTester(self      )
										__lowerCAmelCase							       =     FlaxBertModelTester(self      )
										__lowerCAmelCase							       =     vit_model_tester.prepare_config_and_inputs()
										__lowerCAmelCase							       =     bert_model_tester.prepare_config_and_inputs()

										__lowerCAmelCase       ,      __lowerCAmelCase							       =     vision_config_and_inputs

										__lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase							       =     text_config_and_inputs

										# make sure that cross attention layers are added
										return {
										    "text_config": text_config,
										    "vision_config": vision_config,
										    "pixel_values": pixel_values,
										    "attention_mask": attention_mask,
										    "input_ids": input_ids,
										    "token_type_ids": token_type_ids,
										}
@require_torch
class     _UpperCamelCase    (  lowerCAmelCase__			,unittest.TestCase						):
							'''simple docstring'''







							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_vision_text_pretrained(
										    "hf-internal-testing/tiny-random-clip"       ,  "hf-internal-testing/tiny-bert"       ,  vision_from_pt=__a       ,  text_from_pt=__a       ,  )
										__lowerCAmelCase							       =     13
										__lowerCAmelCase							       =     floats_tensor(
										    [
										        batch_size,
										        model.config.vision_config.num_channels,
										        model.config.vision_config.image_size,
										        model.config.vision_config.image_size,
										    ]      )
										__lowerCAmelCase							       =     ids_tensor([batch_size, 4]       ,  model.config.text_config.vocab_size      )
										__lowerCAmelCase							       =     random_attention_mask([batch_size, 4]      )
										__lowerCAmelCase							       =     {"pixel_values": pixel_values, "input_ids": input_ids, "attention_mask": attention_mask}

										return model, inputs

							def      snake_case							(	self       ,  __a       ,  __a      ):
										__lowerCAmelCase							       =     FlaxCLIPVisionModel(__a      )
										__lowerCAmelCase							       =     FlaxBertModel(__a      )
										return vision_model, text_model



							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     FlaxCLIPVisionModelTester(self      )
										__lowerCAmelCase							       =     FlaxBertModelTester(self      )
										__lowerCAmelCase							       =     clip_model_tester.prepare_config_and_inputs()
										__lowerCAmelCase							       =     bert_model_tester.prepare_config_and_inputs()

										__lowerCAmelCase       ,      __lowerCAmelCase							       =     vision_config_and_inputs

										__lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase							       =     text_config_and_inputs

										# make sure that cross attention layers are added
										return {
										    "text_config": text_config,
										    "vision_config": vision_config,
										    "pixel_values": pixel_values,
										    "attention_mask": attention_mask,
										    "input_ids": input_ids,
										    "token_type_ids": token_type_ids,
										}
@require_flax
@require_vision
class     _UpperCamelCase    (  unittest.TestCase						):
							'''simple docstring'''







							@slow
							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     FlaxVisionTextDualEncoderModel.from_pretrained("clip-italian/clip-italian"       ,  logit_scale_init_value=1.0      )
										__lowerCAmelCase							       =     VisionTextDualEncoderProcessor.from_pretrained("clip-italian/clip-italian"      )

										__lowerCAmelCase							       =     Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"      )
										__lowerCAmelCase							       =     processor(
										    text=["una foto di un gatto", "una foto di un cane"]       ,  images=__a       ,  padding=__a       ,  return_tensors="np"      )

										__lowerCAmelCase							       =     model(**__a      )

										# verify the logits
										self.assertEqual(outputs.logits_per_image.shape       ,  (inputs.pixel_values.shape[0], inputs.input_ids.shape[0])      )
										self.assertEqual(
										    outputs.logits_per_text.shape       ,  (inputs.input_ids.shape[0], inputs.pixel_values.shape[0])       ,  )

										__lowerCAmelCase							       =     np.array([[1.2_2_8_4_7_2_7, 0.3_1_0_4_1_2_2]]      )

										self.assertTrue(np.allclose(outputs.logits_per_image       ,  __a       ,  atol=1e-3      )      )








"""simple docstring"""


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
	from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
	A	:  List[Any]            =     True
	from torch.cuda.amp import autocast


A	:  Any            =     logging.getLogger(__name__)
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       str							 =field(
							    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}			,)
							__UpperCAmelCase		:       Optional[bool]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}						)
							__UpperCAmelCase		:       Optional[bool]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Whether to log verbose messages or not."""}			,)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=2.0			,metadata={"""help""": """Maximum temperature for gumbel softmax."""}						)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=0.5			,metadata={"""help""": """Minimum temperature for gumbel softmax."""}						)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=0.999_995			,metadata={"""help""": """Decay of gumbel temperature during training."""}						)


def _lowerCamelCase      (						_UpperCamelCase       ,					_UpperCamelCase ):

			'''simple docstring'''




			logging.basicConfig(
			    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,					datefmt="%m/%d/%Y %H:%M:%S"       ,					handlers=[logging.StreamHandler(sys.stdout )]       ,					)
			__lowerCAmelCase							       =     logging.WARNING
			if model_args.verbose_logging:
						__lowerCAmelCase							       =     logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank ):
						__lowerCAmelCase							       =     logging.INFO
			logger.setLevel(_UpperCamelCase )
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       str							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}						)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""train"""			,metadata={
							        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
							    }			,)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""validation"""			,metadata={
							        """help""": (
							            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
							        )
							    }			,)
							__UpperCAmelCase		:       Optional[str]							 =field(
							    default="""file"""			,metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}			,)
							__UpperCAmelCase		:       bool							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}						)
							__UpperCAmelCase		:       Optional[int]							 =field(
							    default=1			,metadata={
							        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
							    }			,)
							__UpperCAmelCase		:       Optional[int]							 =field(
							    default=lowerCAmelCase__			,metadata={"""help""": """The number of processes to use for the preprocessing."""}			,)
							__UpperCAmelCase		:       Optional[float]							 =field(
							    default=20.0			,metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}						)
@dataclass
class     _UpperCamelCase    :
							'''simple docstring'''







							__UpperCAmelCase		:       WavaVecaForPreTraining
							__UpperCAmelCase		:       WavaVecaFeatureExtractor
							__UpperCAmelCase		:       Union[bool, str]							 ="longest"
							__UpperCAmelCase		:       Optional[int]							 =None
							__UpperCAmelCase		:       Optional[int]							 =None

							def __call__(	self       ,  __a      ):
										# reformat list to dict and set to pytorch format
										__lowerCAmelCase							       =     self.feature_extractor.pad(
										    __a       ,  max_length=self.max_length       ,  padding=self.padding       ,  pad_to_multiple_of=self.pad_to_multiple_of       ,  return_tensors="pt"       ,  )
										__lowerCAmelCase							       =     self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]      )

										__lowerCAmelCase							       =     batch["input_values"].shape[0]

										# make sure that no loss is computed on padded inputs
										if batch["attention_mask"] is not None:
													# compute real output lengths according to convolution formula
													__lowerCAmelCase							       =     self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1      )      ).to(
													    torch.long      )

													__lowerCAmelCase							       =     torch.zeros(
													    (batch_size, mask_indices_seq_length)       ,  dtype=torch.long       ,  device=batch["input_values"].device      )

													# these two operations makes sure that all values
													# before the output lengths indices are attended to
													__lowerCAmelCase							       =     1
													__lowerCAmelCase							       =     attention_mask.flip([-1]      ).cumsum(-1      ).flip([-1]      ).bool()

										# sample randomly masked indices
										__lowerCAmelCase							       =     _compute_mask_indices(
										    (batch_size, mask_indices_seq_length)       ,  self.model.config.mask_time_prob       ,  self.model.config.mask_time_length       ,  attention_mask=__a       ,  min_masks=2       ,  )

										return batch
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  *__a       ,  __a=1       ,  __a=0       ,  __a=1.0       ,  **__a      ):
										super().__init__(*__a       ,  **__a      )
										__lowerCAmelCase							       =     0
										__lowerCAmelCase							       =     max_gumbel_temp
										__lowerCAmelCase							       =     min_gumbel_temp
										__lowerCAmelCase							       =     gumbel_temp_decay



							def      snake_case							(	self       ,  __a       ,  __a      ):
										model.train()
										__lowerCAmelCase							       =     self._prepare_inputs(__a      )

										if self.use_amp:
													with autocast():
																__lowerCAmelCase							       =     self.compute_loss(__a       ,  __a      )
										else:
													__lowerCAmelCase							       =     self.compute_loss(__a       ,  __a      )

										if self.args.n_gpu > 1 or self.deepspeed:
													if model.module.config.ctc_loss_reduction == "mean":
																__lowerCAmelCase							       =     loss.mean()
													elif model.module.config.ctc_loss_reduction == "sum":
																__lowerCAmelCase							       =     loss.sum() / (inputs["mask_time_indices"]).sum()
													else:
																raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"      )

										if self.args.gradient_accumulation_steps > 1:
													__lowerCAmelCase							       =     loss / self.args.gradient_accumulation_steps

										if self.use_amp:
													self.scaler.scale(__a      ).backward()
										elif self.use_apex:
													with amp.scale_loss(__a       ,  self.optimizer      ) as scaled_loss:
																scaled_loss.backward()
										elif self.deepspeed:
													self.deepspeed.backward(__a      )
										else:
													loss.backward()

										self.num_update_step += 1
										# make sure gumbel softmax temperature is decayed
										if self.args.n_gpu > 1 or self.deepspeed:
													model.module.set_gumbel_temperature(
													    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,  self.min_gumbel_temp      )      )
										else:
													model.set_gumbel_temperature(
													    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,  self.min_gumbel_temp      )      )

										return loss.detach()


def _lowerCamelCase      (						):

			'''simple docstring'''




			__lowerCAmelCase							       =     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

			__lowerCAmelCase       ,      __lowerCAmelCase       ,      __lowerCAmelCase							       =     parser.parse_args_into_dataclasses()
			configure_logger(_UpperCamelCase       ,					_UpperCamelCase )

			# Downloading and loading a dataset from the hub.
			__lowerCAmelCase							       =     load_dataset(data_args.dataset_name       ,					data_args.dataset_config_name       ,					cache_dir=model_args.cache_dir )

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						__lowerCAmelCase							       =     DatasetDict()
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"       ,					cache_dir=model_args.cache_dir       ,					)
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"       ,					cache_dir=model_args.cache_dir       ,					)
			else:
						# make sure only "validation" and "train" keys remain"
						__lowerCAmelCase							       =     DatasetDict()
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split="validation"       ,					cache_dir=model_args.cache_dir       ,					)
						__lowerCAmelCase							       =     load_dataset(
						    data_args.dataset_name       ,					data_args.dataset_config_name       ,					split=f"{data_args.train_split_name}"       ,					cache_dir=model_args.cache_dir       ,					)

			# only normalized-inputs-training is supported
			__lowerCAmelCase							       =     WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path       ,					cache_dir=model_args.cache_dir       ,					do_normalize=_UpperCamelCase )

			def prepare_dataset(_UpperCamelCase ):
						# check that all files have the correct sampling rate
						__lowerCAmelCase       ,      __lowerCAmelCase							       =     librosa.load(batch[data_args.speech_file_column]       ,					sr=feature_extractor.sampling_rate )
						return batch

			# load audio files into numpy arrays
			__lowerCAmelCase							       =     datasets.map(
			    _UpperCamelCase       ,					num_proc=data_args.preprocessing_num_workers       ,					remove_columns=datasets["train"].column_names )

			# filter audio files that are too long
			__lowerCAmelCase							       =     vectorized_datasets.filter(
			    lambda _UpperCamelCase : len(data["speech"] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

			def normalize(_UpperCamelCase ):
						return feature_extractor(batch["speech"]       ,					sampling_rate=feature_extractor.sampling_rate )

			# normalize and transform to `BatchFeatures`
			__lowerCAmelCase							       =     vectorized_datasets.map(
			    _UpperCamelCase       ,					batched=_UpperCamelCase       ,					num_proc=data_args.preprocessing_num_workers       ,					load_from_cache_file=not data_args.overwrite_cache       ,					remove_columns=vectorized_datasets["train"].column_names       ,					)

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			__lowerCAmelCase							       =     WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path       ,					cache_dir=model_args.cache_dir       ,					gradient_checkpointing=training_args.gradient_checkpointing       ,					)

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
						    " ``config.feat_extract_norm='layer'" )

			__lowerCAmelCase							       =     WavaVecaForPreTraining(_UpperCamelCase )

			__lowerCAmelCase							       =     DataCollatorForWavaVecaPretraining(model=_UpperCamelCase       ,					feature_extractor=_UpperCamelCase )

			__lowerCAmelCase							       =     WavaVecaPreTrainer(
			    model=_UpperCamelCase       ,					data_collator=_UpperCamelCase       ,					args=_UpperCamelCase       ,					train_dataset=vectorized_datasets["train"]       ,					eval_dataset=vectorized_datasets["validation"]       ,					tokenizer=_UpperCamelCase       ,					max_gumbel_temp=model_args.max_gumbel_temperature       ,					min_gumbel_temp=model_args.min_gumbel_temperature       ,					gumbel_temp_decay=model_args.gumbel_temperature_decay       ,					)
			trainer.train()


if __name__ == "__main__":
	main()




