






'''simple docstring'''







import operator as op


_A	:			Optional[Any]         ='''scaler.pt'''
_A	:			Optional[Any]         ='''pytorch_model'''
_A	:			int         ='''random_states'''
_A	:			List[Any]         ='''optimizer'''
_A	:			Dict         ='''scheduler'''
_A	:			Dict         ='''pytorch_model.bin'''
_A	:			Optional[Any]         ='''pytorch_model.bin.index.json'''
_A	:			List[str]         ='''model.safetensors'''
_A	:			List[Any]         ='''model.safetensors.index.json'''
_A	:			str         ='''1.10.2'''
_A	:			List[Any]         ='''py38'''
_A	:			int         ='''4.17.0'''
_A	:			List[str]         =['''ml.p3.16xlarge''', '''ml.p3dn.24xlarge''', '''ml.p4dn.24xlarge''']
_A	:			Tuple         =['''FULL_SHARD''', '''SHARD_GRAD_OP''', '''NO_SHARD''', '''HYBRID_SHARD''', '''HYBRID_SHARD_ZERO2''']
_A	:			Tuple         =['''TRANSFORMER_BASED_WRAP''', '''SIZE_BASED_WRAP''', '''NO_WRAP''']
_A	:			Optional[int]         =['''BACKWARD_PRE''', '''BACKWARD_POST''', '''NO_PREFETCH''']
_A	:			Optional[int]         =['''FULL_STATE_DICT''', '''LOCAL_STATE_DICT''', '''SHARDED_STATE_DICT''']
_A	:			List[Any]         ='''2.0.1'''
_A	:			str         =['''pdsh''', '''standard''', '''openmpi''', '''mvapich''']
_A	:			List[str]         =['''default''', '''reduce-overhead''', '''max-autotune''']

_A	:			str         ={'''>''': op.gt, '''>=''': op.ge, '''==''': op.eq, '''!=''': op.ne, '''<=''': op.le, '''<''': op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
_A	:			Tuple         =[
    '''nnodes''',
    '''nproc_per_node''',
    '''rdzv_backend''',
    '''rdzv_endpoint''',
    '''rdzv_id''',
    '''rdzv_conf''',
    '''standalone''',
    '''max_restarts''',
    '''monitor_interval''',
    '''start_method''',
    '''role''',
    '''module''',
    '''m''',
    '''no_python''',
    '''run_path''',
    '''log_dir''',
    '''r''',
    '''redirects''',
    '''t''',
    '''tee''',
    '''node_rank''',
    '''master_addr''',
    '''master_port''',
]

_A	:			Any         =['''DEEPSPEED''', '''MULTI_GPU''', '''FSDP''', '''MEGATRON_LM''']
_A	:			Union[str, Any]         =['''DEEPSPEED''', '''MULTI_XPU''', '''FSDP''']






'''simple docstring'''

import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 	__magic_name__						(    _UpperCamelCase			,						unittest.TestCase   ):
							lowerCAmelCase       :    str   				=					LayoutLMTokenizer
							lowerCAmelCase       :    Tuple   				=					LayoutLMTokenizerFast
							lowerCAmelCase       :    List[Any]   				=					True
							lowerCAmelCase       :    int   				=					True







							def 							__lowercase			(							self					:       Dict		):
									super().setUp()

									_a     :      int        =							[
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    'want',
									    '##want',
									    '##ed',
									    'wa',
									    'un',
									    'runn',
									    '##ing',
									    ',',
									    'low',
									    'lowest',
									]
									_a     :      List[str]        =							os.path.join(self.tmpdirname	,VOCAB_FILES_NAMES['vocab_file']		)
									with open(self.vocab_file	,'w'	,encoding='utf-8'		) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)







							def 							__lowercase			(							self					:       Dict	,**_UpperCAmelCase					:       List[str]		):
									return LayoutLMTokenizer.from_pretrained(self.tmpdirname	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       Tuple		):
									_a     :      Optional[int]        =							'UNwant\u00E9d,running'
									_a     :      List[Any]        =							'unwanted, running'
									return input_text, output_text







							def 							__lowercase			(							self					:       Optional[int]		):
									_a     :      Optional[Any]        =							self.tokenizer_class(self.vocab_file		)

									_a     :      Optional[Any]        =							tokenizer.tokenize('UNwant\u00E9d,running'		)
									self.assertListEqual(_UpperCAmelCase	,['un', '##want', '##ed', ',', 'runn', '##ing']		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase		)	,[7, 4, 5, 10, 8, 9]		)







							def 							__lowercase			(							self					:       Optional[int]		):
									pass




