
'''simple docstring'''




import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class  A				(							__snake_case    ,			unittest.TestCase						):
     __magic_name__						=		LxmertTokenizer
     __magic_name__						=		LxmertTokenizerFast
     __magic_name__						=		True
     __magic_name__						=		True





     def 				__lowerCAmelCase							(							self	) ->				Optional[int]:






         """simple docstring"""






         super().setUp()

         A       :   Optional[int]					=						[
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         A       :   Optional[int]					=						os.path.join(self.tmpdirname						,			VOCAB_FILES_NAMES['''vocab_file''']	)
         with open(self.vocab_file						,			'''w'''						,			encoding='''utf-8'''	) as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				Tuple:






         """simple docstring"""






         A       :   List[Any]					=						'''UNwant\u00E9d,running'''
         A       :   Any					=						'''unwanted, running'''
         return input_text, output_text





     def 				__lowerCAmelCase							(							self	) ->				Dict:






         """simple docstring"""






         A       :   List[Any]					=						self.tokenizer_class(self.vocab_file	)

         A       :   Tuple					=						tokenizer.tokenize('''UNwant\u00E9d,running'''	)
         self.assertListEqual(__SCREAMING_SNAKE_CASE						,			['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE	)						,			[7, 4, 5, 10, 8, 9]	)





     def 				__lowerCAmelCase							(							self	) ->				List[Any]:






         """simple docstring"""






         if not self.test_rust_tokenizer:
             return

         A       :   List[Any]					=						self.get_tokenizer()
         A       :   Union[str, Any]					=						self.get_rust_tokenizer()

         A       :   List[str]					=						'''I was born in 92000, and this is falsé.'''

         A       :   Optional[Any]					=						tokenizer.tokenize(__SCREAMING_SNAKE_CASE	)
         A       :   Optional[Any]					=						rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE	)
         self.assertListEqual(__SCREAMING_SNAKE_CASE						,			__SCREAMING_SNAKE_CASE	)

         A       :   Dict					=						tokenizer.encode(__SCREAMING_SNAKE_CASE						,			add_special_tokens=__SCREAMING_SNAKE_CASE	)
         A       :   str					=						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE						,			add_special_tokens=__SCREAMING_SNAKE_CASE	)
         self.assertListEqual(__SCREAMING_SNAKE_CASE						,			__SCREAMING_SNAKE_CASE	)

         A       :   Any					=						self.get_rust_tokenizer()
         A       :   Optional[int]					=						tokenizer.encode(__SCREAMING_SNAKE_CASE	)
         A       :   List[Any]					=						rust_tokenizer.encode(__SCREAMING_SNAKE_CASE	)
         self.assertListEqual(__SCREAMING_SNAKE_CASE						,			__SCREAMING_SNAKE_CASE	)







'''simple docstring'''



# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


UpperCAmelCase :       Any			   =     [
    'EAGER',
    'AOT_EAGER',
    'INDUCTOR',
    'NVFUSER',
    'AOT_NVFUSER',
    'AOT_CUDAGRAPHS',
    'OFI',
    'FX2TRT',
    'ONNXRT',
    'IPEX',
]







def    a__       (  a__      ,    a__=None      ,    a__=None      ,    a__=None						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      True
     while ask_again:
          __SCREAMING_SNAKE_CASE     =      input(a__						)
          try:
               if default is not None and len(a__						) == 0:
                    return default
               return convert_value(a__						) if convert_value is not None else result
          except Exception:
               if error_message is not None:
                    print(a__						)







def    a__       (  a__      ,    a__=[]      ,    a__=None      ,    a__=0						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      BulletMenu(a__      ,    a__						)
     __SCREAMING_SNAKE_CASE     =      menu.run(default_choice=a__						)
     return convert_value(a__						) if convert_value is not None else result







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return ComputeEnvironment(["""LOCAL_MACHINE""", """AMAZON_SAGEMAKER"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return DistributedType(["""NO""", """MULTI_CPU""", """MULTI_XPU""", """MULTI_GPU""", """MULTI_NPU""", """TPU"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return DynamoBackend(DYNAMO_BACKENDS[value]						).value







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return PrecisionType(["""no""", """fp16""", """bf16""", """fp8"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE     =      int(a__						)
     return SageMakerDistributedType(["""NO""", """DATA_PARALLEL""", """MODEL_PARALLEL"""][value]						)







def    a__       (  a__						):




     """simple docstring"""


     return {"yes": True, "no": False}[value.lower()]







class 						lowerCAmelCase__   (      argparse.RawDescriptionHelpFormatter					):



 """simple docstring"""



 def       UpperCAmelCase__		(		self					:       str					,							__SCREAMING_SNAKE_CASE					:       Optional[Any]					,							__SCREAMING_SNAKE_CASE					:       Dict					,							__SCREAMING_SNAKE_CASE					:       Optional[int]					,							__SCREAMING_SNAKE_CASE					:       List[Any]			)  -> Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE     =      super()._format_usage(__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE					,							__SCREAMING_SNAKE_CASE			)
      __SCREAMING_SNAKE_CASE     =      usage.replace("""<command> [<args>] """					,							""""""			)
      return usage






