



'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


__lowerCAmelCase									=   logging.get_logger(__name__)

__lowerCAmelCase									=   {'''vocab_file''': '''vocab.txt'''}

__lowerCAmelCase									=   {
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

__lowerCAmelCase									=   {
    '''YituTech/conv-bert-base''': 512,
    '''YituTech/conv-bert-medium-small''': 512,
    '''YituTech/conv-bert-small''': 512,
}


__lowerCAmelCase									=   {
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}

class 	__magic_name__						(    _UpperCamelCase   ):
							lowerCAmelCase       :    Union[str, Any]   				=					VOCAB_FILES_NAMES
							lowerCAmelCase       :    Tuple   				=					PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase       :    Any   				=					PRETRAINED_INIT_CONFIGURATION
							lowerCAmelCase       :    Dict   				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase       :    Optional[int]   				=					ConvBertTokenizer







							def __init__(							self					:       Union[str, Any]	,_UpperCAmelCase					:       Tuple=None	,_UpperCAmelCase					:       Optional[Any]=None	,_UpperCAmelCase					:       Any=True	,_UpperCAmelCase					:       Dict="[UNK]"	,_UpperCAmelCase					:       Optional[int]="[SEP]"	,_UpperCAmelCase					:       Optional[Any]="[PAD]"	,_UpperCAmelCase					:       List[str]="[CLS]"	,_UpperCAmelCase					:       List[Any]="[MASK]"	,_UpperCAmelCase					:       Dict=True	,_UpperCAmelCase					:       Optional[int]=None	,**_UpperCAmelCase					:       Dict	,):
									super().__init__(
									    _UpperCAmelCase	,tokenizer_file=_UpperCAmelCase	,do_lower_case=_UpperCAmelCase	,unk_token=_UpperCAmelCase	,sep_token=_UpperCAmelCase	,pad_token=_UpperCAmelCase	,cls_token=_UpperCAmelCase	,mask_token=_UpperCAmelCase	,tokenize_chinese_chars=_UpperCAmelCase	,strip_accents=_UpperCAmelCase	,**_UpperCAmelCase	,)

									_a     :      Union[str, Any]        =							json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
									if (
									    normalizer_state.get('lowercase'	,_UpperCAmelCase		) != do_lower_case
									    or normalizer_state.get('strip_accents'	,_UpperCAmelCase		) != strip_accents
									    or normalizer_state.get('handle_chinese_chars'	,_UpperCAmelCase		) != tokenize_chinese_chars
									):
											_a     :      Optional[Any]        =							getattr(_UpperCAmelCase	,normalizer_state.pop('type'		)		)
											_a     :      Any        =							do_lower_case
											_a     :      int        =							strip_accents
											_a     :      str        =							tokenize_chinese_chars
											_a     :      List[str]        =							normalizer_class(**_UpperCAmelCase		)

									_a     :      Tuple        =							do_lower_case







							def 							__lowercase			(							self					:       Tuple	,_UpperCAmelCase					:       Optional[int]	,_UpperCAmelCase					:       Tuple=None		):
									_a     :      Optional[int]        =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

									return output







							def 							__lowercase			(							self					:       Any	,_UpperCAmelCase					:       List[int]	,_UpperCAmelCase					:       Optional[List[int]] = None		):
									_a     :      Optional[Any]        =							[self.sep_token_id]
									_a     :      Tuple        =							[self.cls_token_id]
									if token_ids_a is None:
											return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







							def 							__lowercase			(							self					:       str	,_UpperCAmelCase					:       str	,_UpperCAmelCase					:       Optional[str] = None		):
									_a     :      Union[str, Any]        =							self._tokenizer.model.save(_UpperCAmelCase	,name=_UpperCAmelCase		)
									return tuple(_UpperCAmelCase		)









'''simple docstring'''

import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 	__magic_name__						(    _UpperCamelCase			,						unittest.TestCase   ):
							lowerCAmelCase       :    str   				=					LayoutLMTokenizer
							lowerCAmelCase       :    Tuple   				=					LayoutLMTokenizerFast
							lowerCAmelCase       :    List[Any]   				=					True
							lowerCAmelCase       :    int   				=					True







							def 							__lowercase			(							self					:       Dict		):
									super().setUp()

									_a     :      int        =							[
									    '[UNK]',
									    '[CLS]',
									    '[SEP]',
									    'want',
									    '##want',
									    '##ed',
									    'wa',
									    'un',
									    'runn',
									    '##ing',
									    ',',
									    'low',
									    'lowest',
									]
									_a     :      List[str]        =							os.path.join(self.tmpdirname	,VOCAB_FILES_NAMES['vocab_file']		)
									with open(self.vocab_file	,'w'	,encoding='utf-8'		) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)







							def 							__lowercase			(							self					:       Dict	,**_UpperCAmelCase					:       List[str]		):
									return LayoutLMTokenizer.from_pretrained(self.tmpdirname	,**_UpperCAmelCase		)







							def 							__lowercase			(							self					:       Optional[Any]	,_UpperCAmelCase					:       Tuple		):
									_a     :      Optional[int]        =							'UNwant\u00E9d,running'
									_a     :      List[Any]        =							'unwanted, running'
									return input_text, output_text







							def 							__lowercase			(							self					:       Optional[int]		):
									_a     :      Optional[Any]        =							self.tokenizer_class(self.vocab_file		)

									_a     :      Optional[Any]        =							tokenizer.tokenize('UNwant\u00E9d,running'		)
									self.assertListEqual(_UpperCAmelCase	,['un', '##want', '##ed', ',', 'runn', '##ing']		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase		)	,[7, 4, 5, 10, 8, 9]		)







							def 							__lowercase			(							self					:       Optional[int]		):
									pass




