





import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType

class 	A_       (a_			):
	UpperCAmelCase__					=			['''image_processor''', '''tokenizer''']
	UpperCAmelCase__					=			'''LayoutLMv2ImageProcessor'''
	UpperCAmelCase__					=			('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')
	def __init__(    self							,					_A=None							,					_A=None							,					**_A		):






				'''simple docstring'''







				if "feature_extractor" in kwargs:
							warnings.warn(
							    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
							    ''' instead.'''							,					_A							,					)
							UpperCAmelCase   		=				kwargs.pop('''feature_extractor'''		)

				UpperCAmelCase   		=				image_processor if image_processor is not None else feature_extractor
				if image_processor is None:
							raise ValueError('''You need to specify an `image_processor`.'''		)
				if tokenizer is None:
							raise ValueError('''You need to specify a `tokenizer`.'''		)

				super().__init__(_A							,					_A		)
	def __call__(    self							,					_A							,					_A = None							,					_A = None							,					_A = None							,					_A = None							,					_A = True							,					_A = False							,					_A = None							,					_A = None							,					_A = 0							,					_A = None							,					_A = None							,					_A = None							,					_A = False							,					_A = False							,					_A = False							,					_A = False							,					_A = True							,					_A = None							,					**_A							,					):






				'''simple docstring'''







				if self.image_processor.apply_ocr and (boxes is not None):
							raise ValueError(
							    '''You cannot provide bounding boxes '''
							    '''if you initialized the image processor with apply_ocr set to True.'''		)

				if self.image_processor.apply_ocr and (word_labels is not None):
							raise ValueError(
							    '''You cannot provide word labels if you initialized the image processor with apply_ocr set to True.'''		)

				if return_overflowing_tokens is True and return_offsets_mapping is False:
							raise ValueError('''You cannot return overflowing tokens without returning the offsets mapping.'''		)

				# first, apply the image processor
				UpperCAmelCase   		=				self.image_processor(images=_A							,					return_tensors=_A		)

				# second, apply the tokenizer
				if text is not None and self.image_processor.apply_ocr and text_pair is None:
							if isinstance(_A							,					_A		):
										UpperCAmelCase   		=				[text]  # add batch dimension (as the image processor always adds a batch dimension)
							UpperCAmelCase   		=				features['''words''']

				UpperCAmelCase   		=				self.tokenizer(
				    text=text if text is not None else features['''words''']							,					text_pair=text_pair if text_pair is not None else None							,					boxes=boxes if boxes is not None else features['''boxes''']							,					word_labels=_A							,					add_special_tokens=_A							,					padding=_A							,					truncation=_A							,					max_length=_A							,					stride=_A							,					pad_to_multiple_of=_A							,					return_token_type_ids=_A							,					return_attention_mask=_A							,					return_overflowing_tokens=_A							,					return_special_tokens_mask=_A							,					return_offsets_mapping=_A							,					return_length=_A							,					verbose=_A							,					return_tensors=_A							,					**_A							,					)

				# add pixel values
				UpperCAmelCase   		=				features.pop('''pixel_values'''		)
				if return_overflowing_tokens is True:
							UpperCAmelCase   		=				self.get_overflowing_images(_A							,					encoded_inputs['''overflow_to_sample_mapping''']		)
				UpperCAmelCase   		=				images

				return encoded_inputs
	def      _lowercase				(    self							,					_A							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				for sample_idx in overflow_to_sample_mapping:
							images_with_overflow.append(images[sample_idx]		)

				if len(_A		) != len(_A		):
							raise ValueError(
							    '''Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got'''
							    F""" {len(_A		)} and {len(_A		)}"""		)

				return images_with_overflow
	def      _lowercase				(    self							,					*_A							,					**_A		):






				'''simple docstring'''







				return self.tokenizer.batch_decode(*_A							,					**_A		)
	def      _lowercase				(    self							,					*_A							,					**_A		):






				'''simple docstring'''







				return self.tokenizer.decode(*_A							,					**_A		)
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				return ["input_ids", "bbox", "attention_mask", "image"]
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				warnings.warn(
				    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''							,					_A							,					)
				return self.image_processor_class







	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				warnings.warn(
				    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''							,					_A							,					)
				return self.image_processor










import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


__A				:    Dict       =   logging.get_logger(__name__)

__A				:    Any       =   {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A				:    Tuple       =   {
    "vocab_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"
        ),
        "squeezebert/squeezebert-mnli": "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt",
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "squeezebert/squeezebert-uncased": (
            "https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli": (
            "https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"
        ),
        "squeezebert/squeezebert-mnli-headless": (
            "https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"
        ),
    },
}

__A				:    List[Any]       =   {
    "squeezebert/squeezebert-uncased": 512,
    "squeezebert/squeezebert-mnli": 512,
    "squeezebert/squeezebert-mnli-headless": 512,
}


__A				:    List[Any]       =   {
    "squeezebert/squeezebert-uncased": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli": {"do_lower_case": True},
    "squeezebert/squeezebert-mnli-headless": {"do_lower_case": True},
}

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_INIT_CONFIGURATION
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase__					=			SqueezeBertTokenizer
	def __init__(    self							,					_A=None							,					_A=None							,					_A=True							,					_A="[UNK]"							,					_A="[SEP]"							,					_A="[PAD]"							,					_A="[CLS]"							,					_A="[MASK]"							,					_A=True							,					_A=None							,					**_A							,					):






				'''simple docstring'''







				super().__init__(
				    _A							,					tokenizer_file=_A							,					do_lower_case=_A							,					unk_token=_A							,					sep_token=_A							,					pad_token=_A							,					cls_token=_A							,					mask_token=_A							,					tokenize_chinese_chars=_A							,					strip_accents=_A							,					**_A							,					)

				UpperCAmelCase   		=				json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
				if (
				    normalizer_state.get('''lowercase'''							,					_A		) != do_lower_case
				    or normalizer_state.get('''strip_accents'''							,					_A		) != strip_accents
				    or normalizer_state.get('''handle_chinese_chars'''							,					_A		) != tokenize_chinese_chars
				):
							UpperCAmelCase   		=				getattr(_A							,					normalizer_state.pop('''type'''		)		)
							UpperCAmelCase   		=				do_lower_case
							UpperCAmelCase   		=				strip_accents
							UpperCAmelCase   		=				tokenize_chinese_chars
							UpperCAmelCase   		=				normalizer_class(**_A		)

				UpperCAmelCase   		=				do_lower_case
	def      _lowercase				(    self							,					_A							,					_A=None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				self._tokenizer.model.save(_A							,					name=_A		)
				return tuple(_A		)



