

"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase  :      int							      =		{
    '''configuration_funnel''': ['''FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FunnelConfig'''],
    '''convert_funnel_original_tf_checkpoint_to_pytorch''': [],
    '''tokenization_funnel''': ['''FunnelTokenizer'''],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase  :      Dict							      =		['''FunnelTokenizerFast''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase  :      int							      =		[
							    '''FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''FunnelBaseModel''',
							    '''FunnelForMaskedLM''',
							    '''FunnelForMultipleChoice''',
							    '''FunnelForPreTraining''',
							    '''FunnelForQuestionAnswering''',
							    '''FunnelForSequenceClassification''',
							    '''FunnelForTokenClassification''',
							    '''FunnelModel''',
							    '''FunnelPreTrainedModel''',
							    '''load_tf_weights_in_funnel''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase  :      Optional[Any]							      =		[
							    '''TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFFunnelBaseModel''',
							    '''TFFunnelForMaskedLM''',
							    '''TFFunnelForMultipleChoice''',
							    '''TFFunnelForPreTraining''',
							    '''TFFunnelForQuestionAnswering''',
							    '''TFFunnelForSequenceClassification''',
							    '''TFFunnelForTokenClassification''',
							    '''TFFunnelModel''',
							    '''TFFunnelPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
							from .tokenization_funnel import FunnelTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_funnel_fast import FunnelTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_funnel import (
														    FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    FunnelBaseModel,
														    FunnelForMaskedLM,
														    FunnelForMultipleChoice,
														    FunnelForPreTraining,
														    FunnelForQuestionAnswering,
														    FunnelForSequenceClassification,
														    FunnelForTokenClassification,
														    FunnelModel,
														    FunnelPreTrainedModel,
														    load_tf_weights_in_funnel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_funnel import (
														    TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFFunnelBaseModel,
														    TFFunnelForMaskedLM,
														    TFFunnelForMultipleChoice,
														    TFFunnelForPreTraining,
														    TFFunnelForQuestionAnswering,
														    TFFunnelForSequenceClassification,
														    TFFunnelForTokenClassification,
														    TFFunnelModel,
														    TFFunnelPreTrainedModel,
														)

else:
							import sys

							__UpperCamelCase  :      Any							      =		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)







"""simple docstring"""





import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__UpperCamelCase  :      Any							      =		logging.getLogger(__name__)

if __name__ == "__main__":
							__UpperCamelCase  :      Union[str, Any]							      =		argparse.ArgumentParser(
							    description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
							)
							parser.add_argument(
							    '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
							)
							parser.add_argument(
							    '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
							)
							parser.add_argument('''--vocab_size''', default=3_0_5_2_2, type=int)
							__UpperCamelCase  :      Any							      =		parser.parse_args()

							logger.info(F'''Loading data from {args.data_file}''')
							with open(args.data_file, '''rb''') as fp:
														__UpperCamelCase  :      List[Any]							      =		pickle.load(fp)

							logger.info('''Counting occurrences for MLM.''')
							__UpperCamelCase  :      Dict							      =		Counter()
							for tk_ids in data:
														counter.update(tk_ids)
							__UpperCamelCase  :      Optional[Any]							      =		[0] * args.vocab_size
							for k, v in counter.items():
														__UpperCamelCase  :      Dict							      =		v

							logger.info(F'''Dump to {args.token_counts_dump}''')
							with open(args.token_counts_dump, '''wb''') as handle:
														pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




