

import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
					from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
					_UpperCAmelCase		:   Tuple										=       True
					from torch.cuda.amp import autocast


_UpperCAmelCase		:   int										=       logging.getLogger(__name__)







@dataclass
class  lowercase				:
				__SCREAMING_SNAKE_CASE					: str       			=			field(
				    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[str]       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}	,  )
				__SCREAMING_SNAKE_CASE					: Optional[bool]       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[bool]       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''Whether to log verbose messages or not.'''}	,  )
				__SCREAMING_SNAKE_CASE					: Optional[float]       			=			field(
				    default=2.0	,  metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[float]       			=			field(
				    default=0.5	,  metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[float]       			=			field(
				    default=0.999995	,  metadata={'''help''': '''Decay of gumbel temperature during training.'''}			)





def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							logging.basicConfig(
							    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'   ,							datefmt='%m/%d/%Y %H:%M:%S'   ,							handlers=[logging.StreamHandler(sys.stdout      )]   ,							)
							snake_case_							=    logging.WARNING
							if model_args.verbose_logging:
														snake_case_							=    logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank      ):
														snake_case_							=    logging.INFO
							logger.setLevel(UpperCamelCase__      )







@dataclass
class  lowercase				:
				__SCREAMING_SNAKE_CASE					: str       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[str]       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[str]       			=			field(
				    default='''train'''	,  metadata={
				        '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
				    }	,  )
				__SCREAMING_SNAKE_CASE					: Optional[str]       			=			field(
				    default='''validation'''	,  metadata={
				        '''help''': (
				            '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
				        )
				    }	,  )
				__SCREAMING_SNAKE_CASE					: Optional[str]       			=			field(
				    default='''file'''	,  metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}	,  )
				__SCREAMING_SNAKE_CASE					: bool       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}			)
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			field(
				    default=1	,  metadata={
				        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
				    }	,  )
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			field(
				    default=lowercase_	,  metadata={'''help''': '''The number of processes to use for the preprocessing.'''}	,  )
				__SCREAMING_SNAKE_CASE					: Optional[float]       			=			field(
				    default=20.0	,  metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}			)







@dataclass
class  lowercase				:
				__SCREAMING_SNAKE_CASE					: WavaVecaForPreTraining
				__SCREAMING_SNAKE_CASE					: WavaVecaFeatureExtractor
				__SCREAMING_SNAKE_CASE					: Union[bool, str]       			=			"longest"
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			None
				__SCREAMING_SNAKE_CASE					: Optional[int]       			=			None




				def __call__(					self      ,   snake_case       ):
											# reformat list to dict and set to pytorch format
											snake_case_							=    self.feature_extractor.pad(
											    snake_case      ,   max_length=self.max_length      ,   padding=self.padding      ,   pad_to_multiple_of=self.pad_to_multiple_of      ,   return_tensors='pt'      ,   )
											snake_case_							=    self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1]       )

											snake_case_							=    batch['input_values'].shape[0]

											# make sure that no loss is computed on padded inputs
											if batch["attention_mask"] is not None:
																		# compute real output lengths according to convolution formula
																		snake_case_							=    self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1       )       ).to(
																		    torch.long       )

																		snake_case_							=    torch.zeros(
																		    (batch_size, mask_indices_seq_length)      ,   dtype=torch.long      ,   device=batch['input_values'].device       )

																		# these two operations makes sure that all values
																		# before the output lengths indices are attended to
																		snake_case_							=    1
																		snake_case_							=    attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

											# sample randomly masked indices
											snake_case_							=    _compute_mask_indices(
											    (batch_size, mask_indices_seq_length)      ,   self.model.config.mask_time_prob      ,   self.model.config.mask_time_length      ,   attention_mask=snake_case      ,   min_masks=2      ,   )

											return batch







class  lowercase				(       lowercase_			):




				def __init__(					self      ,   *snake_case      ,   snake_case=1      ,   snake_case=0      ,   snake_case=1.0      ,   **snake_case       ):
											super().__init__(*snake_case      ,   **snake_case       )
											snake_case_							=    0
											snake_case_							=    max_gumbel_temp
											snake_case_							=    min_gumbel_temp
											snake_case_							=    gumbel_temp_decay




				def     a							(					self      ,   snake_case      ,   snake_case       ):
											model.train()
											snake_case_							=    self._prepare_inputs(snake_case       )

											if self.use_amp:
																		with autocast():
																									snake_case_							=    self.compute_loss(snake_case      ,   snake_case       )
											else:
																		snake_case_							=    self.compute_loss(snake_case      ,   snake_case       )

											if self.args.n_gpu > 1 or self.deepspeed:
																		if model.module.config.ctc_loss_reduction == "mean":
																									snake_case_							=    loss.mean()
																		elif model.module.config.ctc_loss_reduction == "sum":
																									snake_case_							=    loss.sum() / (inputs['mask_time_indices']).sum()
																		else:
																									raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''       )

											if self.args.gradient_accumulation_steps > 1:
																		snake_case_							=    loss / self.args.gradient_accumulation_steps

											if self.use_amp:
																		self.scaler.scale(snake_case       ).backward()
											elif self.use_apex:
																		with amp.scale_loss(snake_case      ,   self.optimizer       ) as scaled_loss:
																									scaled_loss.backward()
											elif self.deepspeed:
																		self.deepspeed.backward(snake_case       )
											else:
																		loss.backward()

											self.num_update_step += 1
											# make sure gumbel softmax temperature is decayed
											if self.args.n_gpu > 1 or self.deepspeed:
																		model.module.set_gumbel_temperature(
																		    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,   self.min_gumbel_temp       )       )
											else:
																		model.set_gumbel_temperature(
																		    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,   self.min_gumbel_temp       )       )

											return loss.detach()





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)      )

							snake_case_				,    snake_case_				,    snake_case_							=    parser.parse_args_into_dataclasses()
							configure_logger(UpperCamelCase__   ,							UpperCamelCase__      )

							# Downloading and loading a dataset from the hub.
							snake_case_							=    load_dataset(data_args.dataset_name   ,							data_args.dataset_config_name   ,							cache_dir=model_args.cache_dir      )

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														snake_case_							=    DatasetDict()
														snake_case_							=    load_dataset(
														    data_args.dataset_name   ,							data_args.dataset_config_name   ,							split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''   ,							cache_dir=model_args.cache_dir   ,							)
														snake_case_							=    load_dataset(
														    data_args.dataset_name   ,							data_args.dataset_config_name   ,							split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''   ,							cache_dir=model_args.cache_dir   ,							)
							else:
														# make sure only "validation" and "train" keys remain"
														snake_case_							=    DatasetDict()
														snake_case_							=    load_dataset(
														    data_args.dataset_name   ,							data_args.dataset_config_name   ,							split='validation'   ,							cache_dir=model_args.cache_dir   ,							)
														snake_case_							=    load_dataset(
														    data_args.dataset_name   ,							data_args.dataset_config_name   ,							split=F'''{data_args.train_split_name}'''   ,							cache_dir=model_args.cache_dir   ,							)

							# only normalized-inputs-training is supported
							snake_case_							=    WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path   ,							cache_dir=model_args.cache_dir   ,							do_normalize=UpperCamelCase__      )

							def prepare_dataset(UpperCamelCase__      ):
														# check that all files have the correct sampling rate
														snake_case_				,    snake_case_							=    librosa.load(batch[data_args.speech_file_column]   ,							sr=feature_extractor.sampling_rate      )
														return batch

							# load audio files into numpy arrays
							snake_case_							=    datasets.map(
							    UpperCamelCase__   ,							num_proc=data_args.preprocessing_num_workers   ,							remove_columns=datasets['train'].column_names      )

							# filter audio files that are too long
							snake_case_							=    vectorized_datasets.filter(
							    lambda UpperCamelCase__      : len(data['speech']      ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate      )      )

							def normalize(UpperCamelCase__      ):
														return feature_extractor(batch['speech']   ,							sampling_rate=feature_extractor.sampling_rate      )

							# normalize and transform to `BatchFeatures`
							snake_case_							=    vectorized_datasets.map(
							    UpperCamelCase__   ,							batched=UpperCamelCase__   ,							num_proc=data_args.preprocessing_num_workers   ,							load_from_cache_file=not data_args.overwrite_cache   ,							remove_columns=vectorized_datasets['train'].column_names   ,							)

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							snake_case_							=    WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path   ,							cache_dir=model_args.cache_dir   ,							gradient_checkpointing=training_args.gradient_checkpointing   ,							)

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
														    ' ``config.feat_extract_norm=\'layer\''      )

							snake_case_							=    WavaVecaForPreTraining(UpperCamelCase__      )

							snake_case_							=    DataCollatorForWavaVecaPretraining(model=UpperCamelCase__   ,							feature_extractor=UpperCamelCase__      )

							snake_case_							=    WavaVecaPreTrainer(
							    model=UpperCamelCase__   ,							data_collator=UpperCamelCase__   ,							args=UpperCamelCase__   ,							train_dataset=vectorized_datasets['train']   ,							eval_dataset=vectorized_datasets['validation']   ,							tokenizer=UpperCamelCase__   ,							max_gumbel_temp=model_args.max_gumbel_temperature   ,							min_gumbel_temp=model_args.min_gumbel_temperature   ,							gumbel_temp_decay=model_args.gumbel_temperature_decay   ,							)
							trainer.train()


if __name__ == "__main__":
					main()







import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available


if is_vision_available():
					from PIL import Image

					from transformers import ChineseCLIPImageProcessor, ChineseCLIPProcessor







@require_vision
class  lowercase				(       unittest.TestCase			):




				def     a							(					self       ):
											snake_case_							=    tempfile.mkdtemp()

											snake_case_							=    [
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    '的',
											    '价',
											    '格',
											    '是',
											    '15',
											    '便',
											    'alex',
											    '##andra',
											    '，',
											    '。',
											    '-',
											    't',
											    'shirt',
											]
											snake_case_							=    os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']       )
											with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )

											snake_case_							=    {
											    'do_resize': True,
											    'size': {'height': 224, 'width': 224},
											    'do_center_crop': True,
											    'crop_size': {'height': 18, 'width': 18},
											    'do_normalize': True,
											    'image_mean': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
											    'image_std': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
											    'do_convert_rgb': True,
											}
											snake_case_							=    os.path.join(self.tmpdirname      ,   snake_case       )
											with open(self.image_processor_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		json.dump(snake_case      ,   snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return BertTokenizer.from_pretrained(self.tmpdirname      ,   **snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return BertTokenizerFast.from_pretrained(self.tmpdirname      ,   **snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return ChineseCLIPImageProcessor.from_pretrained(self.tmpdirname      ,   **snake_case       )




				def     a							(					self       ):
											shutil.rmtree(self.tmpdirname       )




				def     a							(					self       ):
											snake_case_							=    [np.random.randint(255      ,   size=(3, 30, 400)      ,   dtype=np.uinta       )]

											snake_case_							=    [Image.fromarray(np.moveaxis(snake_case      ,   0      ,   -1       )       ) for x in image_inputs]

											return image_inputs




				def     a							(					self       ):
											snake_case_							=    self.get_tokenizer()
											snake_case_							=    self.get_rust_tokenizer()
											snake_case_							=    self.get_image_processor()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )
											processor_slow.save_pretrained(self.tmpdirname       )
											snake_case_							=    ChineseCLIPProcessor.from_pretrained(self.tmpdirname      ,   use_fast=snake_case       )

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )
											processor_fast.save_pretrained(self.tmpdirname       )
											snake_case_							=    ChineseCLIPProcessor.from_pretrained(self.tmpdirname       )

											self.assertEqual(processor_slow.tokenizer.get_vocab()      ,   tokenizer_slow.get_vocab()       )
											self.assertEqual(processor_fast.tokenizer.get_vocab()      ,   tokenizer_fast.get_vocab()       )
											self.assertEqual(tokenizer_slow.get_vocab()      ,   tokenizer_fast.get_vocab()       )
											self.assertIsInstance(processor_slow.tokenizer      ,   snake_case       )
											self.assertIsInstance(processor_fast.tokenizer      ,   snake_case       )

											self.assertEqual(processor_slow.image_processor.to_json_string()      ,   image_processor.to_json_string()       )
											self.assertEqual(processor_fast.image_processor.to_json_string()      ,   image_processor.to_json_string()       )
											self.assertIsInstance(processor_slow.image_processor      ,   snake_case       )
											self.assertIsInstance(processor_fast.image_processor      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    ChineseCLIPProcessor(tokenizer=self.get_tokenizer()      ,   image_processor=self.get_image_processor()       )
											processor.save_pretrained(self.tmpdirname       )

											snake_case_							=    self.get_tokenizer(cls_token='(CLS)'      ,   sep_token='(SEP)'       )
											snake_case_							=    self.get_image_processor(do_normalize=snake_case       )

											snake_case_							=    ChineseCLIPProcessor.from_pretrained(
											    self.tmpdirname      ,   cls_token='(CLS)'      ,   sep_token='(SEP)'      ,   do_normalize=snake_case       )

											self.assertEqual(processor.tokenizer.get_vocab()      ,   tokenizer_add_kwargs.get_vocab()       )
											self.assertIsInstance(processor.tokenizer      ,   snake_case       )

											self.assertEqual(processor.image_processor.to_json_string()      ,   image_processor_add_kwargs.to_json_string()       )
											self.assertIsInstance(processor.image_processor      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    image_processor(snake_case      ,   return_tensors='np'       )
											snake_case_							=    processor(images=snake_case      ,   return_tensors='np'       )

											for key in input_feat_extract.keys():
																		self.assertAlmostEqual(input_feat_extract[key].sum()      ,   input_processor[key].sum()      ,   delta=1e-2       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    'Alexandra，T-shirt的价格是15便士。'

											snake_case_							=    processor(text=snake_case       )

											snake_case_							=    tokenizer(snake_case       )

											for key in encoded_tok.keys():
																		self.assertListEqual(encoded_tok[key]      ,   encoded_processor[key]       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    'Alexandra，T-shirt的价格是15便士。'
											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    processor(text=snake_case      ,   images=snake_case       )

											self.assertListEqual(list(inputs.keys()       )      ,   ['input_ids', 'token_type_ids', 'attention_mask', 'pixel_values']       )

											# test if it raises when no input is passed
											with pytest.raises(snake_case       ):
																		processor()




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											snake_case_							=    processor.batch_decode(snake_case       )
											snake_case_							=    tokenizer.batch_decode(snake_case       )

											self.assertListEqual(snake_case      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    ChineseCLIPProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    'Alexandra，T-shirt的价格是15便士。'
											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    processor(text=snake_case      ,   images=snake_case       )

											self.assertListEqual(list(inputs.keys()       )      ,   processor.model_input_names       )




