

"""simple docstring"""





import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__UpperCamelCase  :      Any							      =		logging.getLogger(__name__)

if __name__ == "__main__":
							__UpperCamelCase  :      Union[str, Any]							      =		argparse.ArgumentParser(
							    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
							)
							parser.add_argument(
							    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
							)
							parser.add_argument(
							    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
							)
							parser.add_argument('''--vocab_size''', default=3_0_5_2_2, type=int)
							__UpperCamelCase  :      Any							      =		parser.parse_args()

							logger.info(F'''Loading data from {args.data_file}''')
							with open(args.data_file, '''rb''') as fp:
														__UpperCamelCase  :      List[Any]							      =		pickle.load(fp)

							logger.info('''Counting occurrences for MLM.''')
							__UpperCamelCase  :      Dict							      =		Counter()
							for tk_ids in data:
														counter.update(tk_ids)
							__UpperCamelCase  :      Optional[Any]							      =		[0] * args.vocab_size
							for k, v in counter.items():
														__UpperCamelCase  :      Dict							      =		v

							logger.info(F'''Dump to {args.token_counts_dump}''')
							with open(args.token_counts_dump, '''wb''') as handle:
														pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


__A				:    Union[str, Any]       =   {
    "configuration_speech_to_text": ["SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP", "Speech2TextConfig"],
    "processing_speech_to_text": ["Speech2TextProcessor"],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextTokenizer"]

try:
				if not is_speech_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextFeatureExtractor"]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    int       =   [
				    "TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFSpeech2TextForConditionalGeneration",
				    "TFSpeech2TextModel",
				    "TFSpeech2TextPreTrainedModel",
				]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    Tuple       =   [
				    "SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "Speech2TextForConditionalGeneration",
				    "Speech2TextModel",
				    "Speech2TextPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
				from .processing_speech_to_text import SpeechaTextProcessor

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_speech_to_text import SpeechaTextTokenizer

				try:
								if not is_speech_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_speech_to_text import (
								    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFSpeechaTextForConditionalGeneration,
								    TFSpeechaTextModel,
								    TFSpeechaTextPreTrainedModel,
								)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_speech_to_text import (
								    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SpeechaTextForConditionalGeneration,
								    SpeechaTextModel,
								    SpeechaTextPreTrainedModel,
								)

else:
				import sys

				__A				:    Dict       =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



