

"""simple docstring"""





import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					lowercase__				 =       "Speech2TextFeatureExtractor"
					lowercase__				 =       "Speech2TextTokenizer"







					def __init__(       self			:				Dict   ,lowercase_			:				Optional[Any]   ,lowercase_			:				int		):
						super().__init__(lowercase_   ,lowercase_		)
						lowerCAmelCase__	:						Optional[int]  				=	self.feature_extractor
						lowerCAmelCase__	:						List[str]  				=	False







					def __call__(       self			:				Union[str, Any]   ,*lowercase_			:				Any   ,**lowercase_			:				Any		):
						# For backward compatibility
						if self._in_target_context_manager:
							return self.current_processor(*lowercase_   ,**lowercase_		)

						if "raw_speech" in kwargs:
							warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''		)
							lowerCAmelCase__	:						Tuple  				=	kwargs.pop('''raw_speech'''		)
						else:
							lowerCAmelCase__	:						str  				=	kwargs.pop('''audio'''   ,lowercase_		)
						lowerCAmelCase__	:						Optional[Any]  				=	kwargs.pop('''sampling_rate'''   ,lowercase_		)
						lowerCAmelCase__	:						int  				=	kwargs.pop('''text'''   ,lowercase_		)
						if len(lowercase_		) > 0:
							lowerCAmelCase__	:						int  				=	args[0]
							lowerCAmelCase__	:						Optional[int]  				=	args[1:]

						if audio is None and text is None:
							raise ValueError('''You need to specify either an `audio` or `text` input to process.'''		)

						if audio is not None:
							lowerCAmelCase__	:						Optional[Any]  				=	self.feature_extractor(lowercase_   ,*lowercase_   ,sampling_rate=lowercase_   ,**lowercase_		)
						if text is not None:
							lowerCAmelCase__	:						Optional[int]  				=	self.tokenizer(lowercase_   ,**lowercase_		)

						if text is None:
							return inputs
						elif audio is None:
							return encodings
						else:
							lowerCAmelCase__	:						str  				=	encodings['''input_ids''']
							return inputs







					def        __lowerCAmelCase (       self			:				List[Any]   ,*lowercase_			:				Dict   ,**lowercase_			:				Any		):
						return self.tokenizer.batch_decode(*lowercase_   ,**lowercase_		)







					def        __lowerCAmelCase (       self			:				Dict   ,*lowercase_			:				Optional[int]   ,**lowercase_			:				Union[str, Any]		):
						return self.tokenizer.decode(*lowercase_   ,**lowercase_		)







					@contextmanager
					def        __lowerCAmelCase (       self			:				Tuple		):
						warnings.warn(
						    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
						    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
						    '''your audio inputs, or in a separate call.'''		)
						lowerCAmelCase__	:						int  				=	True
						lowerCAmelCase__	:						List[str]  				=	self.tokenizer
						yield
						lowerCAmelCase__	:						Union[str, Any]  				=	self.feature_extractor
						lowerCAmelCase__	:						Dict  				=	False







"""simple docstring"""





import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__UpperCamelCase  :      Any							      =		logging.getLogger(__name__)

if __name__ == "__main__":
							__UpperCamelCase  :      Union[str, Any]							      =		argparse.ArgumentParser(
							    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
							)
							parser.add_argument(
							    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
							)
							parser.add_argument(
							    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
							)
							parser.add_argument('''--vocab_size''', default=3_0_5_2_2, type=int)
							__UpperCamelCase  :      Any							      =		parser.parse_args()

							logger.info(F'''Loading data from {args.data_file}''')
							with open(args.data_file, '''rb''') as fp:
														__UpperCamelCase  :      List[Any]							      =		pickle.load(fp)

							logger.info('''Counting occurrences for MLM.''')
							__UpperCamelCase  :      Dict							      =		Counter()
							for tk_ids in data:
														counter.update(tk_ids)
							__UpperCamelCase  :      Optional[Any]							      =		[0] * args.vocab_size
							for k, v in counter.items():
														__UpperCamelCase  :      Dict							      =		v

							logger.info(F'''Dump to {args.token_counts_dump}''')
							with open(args.token_counts_dump, '''wb''') as handle:
														pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




