






from argparse import ArgumentParser

from . import BaseTransformersCLICommand

def     snake_case   (UpperCAmelCase__    )			->							Dict:
  return DownloadCommand(args.model      ,       args.cache_dir      ,       args.force      ,       args.trust_remote_code    )




class 							_lowerCAmelCase(			UpperCAmelCase_   ):





       """simple docstring"""







       @staticmethod
       def      _a       (   _lowerCamelCase				):
         UpperCamelCase_:   str        =							parser.add_parser('download'				)
         download_parser.add_argument(
             '--cache-dir'    ,     type=_lowerCamelCase    ,     default=_lowerCamelCase    ,     help='Path to location to store the models'				)
         download_parser.add_argument(
             '--force'    ,     action='store_true'    ,     help='Force the model to be download even if already in cache-dir'				)
         download_parser.add_argument(
             '--trust-remote-code'    ,     action='store_true'    ,     help='Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you\'ve reviewed the code as it will execute on your local machine'    ,     )
         download_parser.add_argument('model'    ,     type=_lowerCamelCase    ,     help='Name of the model to download'				)
         download_parser.set_defaults(func=_lowerCamelCase				)





       def __init__(   self    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase				):
         UpperCamelCase_:   Union[str, Any]        =							model
         UpperCamelCase_:   Optional[int]        =							cache
         UpperCamelCase_:   Tuple        =							force
         UpperCamelCase_:   Union[str, Any]        =							trust_remote_code





       def      _a       (   self				):
         from ..models.auto import AutoModel, AutoTokenizer

         AutoModel.from_pretrained(
             self._model    ,     cache_dir=self._cache    ,     force_download=self._force    ,     trust_remote_code=self._trust_remote_code				)
         AutoTokenizer.from_pretrained(
             self._model    ,     cache_dir=self._cache    ,     force_download=self._force    ,     trust_remote_code=self._trust_remote_code				)





"""simple docstring"""






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class lowerCAmelCase__      (	lowercase,      unittest.TestCase		):






			'''simple docstring'''



			lowerCamelCase__ =     MobileBertTokenizer
			lowerCamelCase__ =     MobileBertTokenizerFast
			lowerCamelCase__ =     True
			lowerCamelCase__ =     True
			lowerCamelCase__ =     filter_non_english
			lowerCamelCase__ =     """google/mobilebert-uncased"""
			def    A_       (    self    ):
							super().setUp()

							_lowerCamelCase	:     Optional[int]       =       [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							_lowerCamelCase	:     Union[str, Any]       =       os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES['vocab_file']    )
							with open(self.vocab_file    , 'w'    , encoding='utf-8'    ) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

							_lowerCamelCase	:     Any       =       [
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]
			def    A_       (    self    , lowercase    ):
							_lowerCamelCase	:     Union[str, Any]       =       'UNwant\u00E9d,running'
							_lowerCamelCase	:     List[Any]       =       'unwanted, running'
							return input_text, output_text
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       self.tokenizer_class(self.vocab_file    )

							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.tokenize('UNwant\u00E9d,running'    )
							self.assertListEqual(lowercase    , ['un', '##want', '##ed', ',', 'runn', '##ing']    )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase    )    , [9, 6, 7, 12, 10, 11]    )
			def    A_       (    self    ):
							if not self.test_rust_tokenizer:
											return

							_lowerCamelCase	:     Any       =       self.get_tokenizer()
							_lowerCamelCase	:     Any       =       self.get_rust_tokenizer()

							_lowerCamelCase	:     int       =       'UNwant\u00E9d,running'

							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.tokenize(lowercase    )
							_lowerCamelCase	:     List[Any]       =       rust_tokenizer.tokenize(lowercase    )
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Dict       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Dict       =       rust_tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Optional[Any]       =       self.get_rust_tokenizer()
							_lowerCamelCase	:     Optional[int]       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     List[str]       =       rust_tokenizer.encode(lowercase    )
							self.assertListEqual(lowercase    , lowercase    )

							# With lower casing
							_lowerCamelCase	:     List[Any]       =       self.get_tokenizer(do_lower_case=lowercase    )
							_lowerCamelCase	:     int       =       self.get_rust_tokenizer(do_lower_case=lowercase    )

							_lowerCamelCase	:     Optional[Any]       =       'UNwant\u00E9d,running'

							_lowerCamelCase	:     Dict       =       tokenizer.tokenize(lowercase    )
							_lowerCamelCase	:     int       =       rust_tokenizer.tokenize(lowercase    )
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Dict       =       tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Dict       =       rust_tokenizer.encode(lowercase    , add_special_tokens=lowercase    )
							self.assertListEqual(lowercase    , lowercase    )

							_lowerCamelCase	:     Any       =       self.get_rust_tokenizer()
							_lowerCamelCase	:     Union[str, Any]       =       tokenizer.encode(lowercase    )
							_lowerCamelCase	:     Dict       =       rust_tokenizer.encode(lowercase    )
							self.assertListEqual(lowercase    , lowercase    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Dict       =       BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )    , ['ah', '\u535A', '\u63A8', 'zz']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Optional[Any]       =       BasicTokenizer(do_lower_case=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    , ['hello', '!', 'how', 'are', 'you', '?']    )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    , ['hello']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[str]       =       BasicTokenizer(do_lower_case=lowercase    , strip_accents=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    , ['hällo', '!', 'how', 'are', 'you', '?']    )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    , ['h\u00E9llo']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[Any]       =       BasicTokenizer(do_lower_case=lowercase    , strip_accents=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    , ['hallo', '!', 'how', 'are', 'you', '?']    )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    , ['hello']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     int       =       BasicTokenizer(do_lower_case=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    , ['hallo', '!', 'how', 'are', 'you', '?']    )
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    , ['hello']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[str]       =       BasicTokenizer(do_lower_case=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    , ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       BasicTokenizer(do_lower_case=lowercase    , strip_accents=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    , ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       BasicTokenizer(do_lower_case=lowercase    , strip_accents=lowercase    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    , ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     int       =       BasicTokenizer(do_lower_case=lowercase    , never_split=['[UNK]']    )

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )    , ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							_lowerCamelCase	:     Tuple       =       {}
							for i, token in enumerate(lowercase    ):
											_lowerCamelCase	:     Union[str, Any]       =       i
							_lowerCamelCase	:     str       =       WordpieceTokenizer(vocab=lowercase    , unk_token='[UNK]'    )

							self.assertListEqual(tokenizer.tokenize(''    )    , []    )

							self.assertListEqual(tokenizer.tokenize('unwanted running'    )    , ['un', '##want', '##ed', 'runn', '##ing']    )

							self.assertListEqual(tokenizer.tokenize('unwantedX running'    )    , ['[UNK]', 'runn', '##ing']    )
			def    A_       (    self    ):
							self.assertTrue(_is_whitespace(' '    )    )
							self.assertTrue(_is_whitespace('\t'    )    )
							self.assertTrue(_is_whitespace('\r'    )    )
							self.assertTrue(_is_whitespace('\n'    )    )
							self.assertTrue(_is_whitespace('\u00A0'    )    )

							self.assertFalse(_is_whitespace('A'    )    )
							self.assertFalse(_is_whitespace('-'    )    )
			def    A_       (    self    ):
							self.assertTrue(_is_control('\u0005'    )    )

							self.assertFalse(_is_control('A'    )    )
							self.assertFalse(_is_control(' '    )    )
							self.assertFalse(_is_control('\t'    )    )
							self.assertFalse(_is_control('\r'    )    )
			def    A_       (    self    ):
							self.assertTrue(_is_punctuation('-'    )    )
							self.assertTrue(_is_punctuation('$'    )    )
							self.assertTrue(_is_punctuation('`'    )    )
							self.assertTrue(_is_punctuation('.'    )    )

							self.assertFalse(_is_punctuation('A'    )    )
							self.assertFalse(_is_punctuation(' '    )    )
			def    A_       (    self    ):
							_lowerCamelCase	:     List[Any]       =       self.get_tokenizer()
							_lowerCamelCase	:     Union[str, Any]       =       self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(lowercase    ) for t in ['Test', '\xad', 'test']]    , [['[UNK]'], [], ['[UNK]']]    )

							self.assertListEqual(
							    [rust_tokenizer.tokenize(lowercase    ) for t in ['Test', '\xad', 'test']]    , [['[UNK]'], [], ['[UNK]']]    )
			@slow
			def    A_       (    self    ):
							_lowerCamelCase	:     Optional[Any]       =       self.tokenizer_class.from_pretrained('google/mobilebert-uncased'    )

							_lowerCamelCase	:     List[str]       =       tokenizer.encode('sequence builders'    , add_special_tokens=lowercase    )
							_lowerCamelCase	:     Dict       =       tokenizer.encode('multi-sequence build'    , add_special_tokens=lowercase    )

							_lowerCamelCase	:     Optional[int]       =       tokenizer.build_inputs_with_special_tokens(lowercase    )
							_lowerCamelCase	:     List[Any]       =       tokenizer.build_inputs_with_special_tokens(lowercase    , lowercase    )

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]
			def    A_       (    self    ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     int       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )

															_lowerCamelCase	:     int       =       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
															_lowerCamelCase	:     int       =       tokenizer_r.encode_plus(
															    lowercase    , return_attention_mask=lowercase    , return_token_type_ids=lowercase    , return_offsets_mapping=lowercase    , add_special_tokens=lowercase    , )

															_lowerCamelCase	:     Tuple       =       tokenizer_r.do_lower_case if hasattr(lowercase    , 'do_lower_case'    ) else False
															_lowerCamelCase	:     Dict       =       (
															    [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), 'A'),
															        ((1, 2), ','),
															        ((3, 5), 'na'),
															        ((5, 6), '##ï'),
															        ((6, 8), '##ve'),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), 'Allen'),
															        ((21, 23), '##NL'),
															        ((23, 24), '##P'),
															        ((25, 33), 'sentence'),
															        ((33, 34), '.'),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															    if not do_lower_case
															    else [
															        ((0, 0), tokenizer_r.cls_token),
															        ((0, 1), 'a'),
															        ((1, 2), ','),
															        ((3, 8), 'naive'),
															        ((9, 15), tokenizer_r.mask_token),
															        ((16, 21), 'allen'),
															        ((21, 23), '##nl'),
															        ((23, 24), '##p'),
															        ((25, 33), 'sentence'),
															        ((33, 34), '.'),
															        ((0, 0), tokenizer_r.sep_token),
															    ]
															)

															self.assertEqual(
															    [e[1] for e in expected_results]    , tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
															self.assertEqual([e[0] for e in expected_results]    , tokens['offset_mapping']    )






			def    A_       (    self    ):
							_lowerCamelCase	:     Tuple       =       ['的', '人', '有']
							_lowerCamelCase	:     Optional[Any]       =       ''.join(lowercase    )
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
															_lowerCamelCase	:     int       =       True
															_lowerCamelCase	:     Any       =       self.tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )

															_lowerCamelCase	:     List[str]       =       tokenizer_p.encode(lowercase    , add_special_tokens=lowercase    )
															_lowerCamelCase	:     List[Any]       =       tokenizer_r.encode(lowercase    , add_special_tokens=lowercase    )

															_lowerCamelCase	:     Any       =       tokenizer_r.convert_ids_to_tokens(lowercase    )
															_lowerCamelCase	:     Dict       =       tokenizer_p.convert_ids_to_tokens(lowercase    )

															# it is expected that each Chinese character is not preceded by "##"
															self.assertListEqual(lowercase    , lowercase    )
															self.assertListEqual(lowercase    , lowercase    )

															_lowerCamelCase	:     Optional[Any]       =       False
															_lowerCamelCase	:     Dict       =       self.rust_tokenizer_class.from_pretrained(lowercase    , **lowercase    )
															_lowerCamelCase	:     Optional[int]       =       self.tokenizer_class.from_pretrained(lowercase    , **lowercase    )

															_lowerCamelCase	:     Optional[Any]       =       tokenizer_r.encode(lowercase    , add_special_tokens=lowercase    )
															_lowerCamelCase	:     str       =       tokenizer_p.encode(lowercase    , add_special_tokens=lowercase    )

															_lowerCamelCase	:     List[Any]       =       tokenizer_r.convert_ids_to_tokens(lowercase    )
															_lowerCamelCase	:     Dict       =       tokenizer_p.convert_ids_to_tokens(lowercase    )

															# it is expected that only the first Chinese character is not preceded by "##".
															_lowerCamelCase	:     List[str]       =       [
															    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowercase    )
															]
															self.assertListEqual(lowercase    , lowercase    )
															self.assertListEqual(lowercase    , lowercase    )