
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__		:List[Any]   =    logging.get_logger(__name__)

lowercase__		:Optional[Any]   =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__		:Union[str, Any]   =    {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__		:str   =    {
    "bert-base-uncased": 512,
    "bert-large-uncased": 512,
    "bert-base-cased": 512,
    "bert-large-cased": 512,
    "bert-base-multilingual-uncased": 512,
    "bert-base-multilingual-cased": 512,
    "bert-base-chinese": 512,
    "bert-base-german-cased": 512,
    "bert-large-uncased-whole-word-masking": 512,
    "bert-large-cased-whole-word-masking": 512,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
    "bert-base-cased-finetuned-mrpc": 512,
    "bert-base-german-dbmdz-cased": 512,
    "bert-base-german-dbmdz-uncased": 512,
    "TurkuNLP/bert-base-finnish-cased-v1": 512,
    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
    "wietsedv/bert-base-dutch-cased": 512,
}

lowercase__		:Dict   =    {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       str       					=VOCAB_FILES_NAMES
							lowercase_       :       Dict       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       Optional[int]       					=PRETRAINED_INIT_CONFIGURATION
							lowercase_       :       Tuple       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Any       					=BertTokenizer




							def __init__(				self   ,A__=None   ,A__=None   ,A__=True   ,A__="[UNK]"   ,A__="[SEP]"   ,A__="[PAD]"   ,A__="[CLS]"   ,A__="[MASK]"   ,A__=True   ,A__=None   ,**A__   ,):
												super().__init__(
												    A__   ,tokenizer_file=A__   ,do_lower_case=A__   ,unk_token=A__   ,sep_token=A__   ,pad_token=A__   ,cls_token=A__   ,mask_token=A__   ,tokenize_chinese_chars=A__   ,strip_accents=A__   ,**A__   ,)

												lowercase							=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
												if (
												    normalizer_state.get('''lowercase'''   ,A__) != do_lower_case
												    or normalizer_state.get('''strip_accents'''   ,A__) != strip_accents
												    or normalizer_state.get('''handle_chinese_chars'''   ,A__) != tokenize_chinese_chars
												):
																	lowercase							=						getattr(A__   ,normalizer_state.pop('''type'''))
																	lowercase							=						do_lower_case
																	lowercase							=						strip_accents
																	lowercase							=						tokenize_chinese_chars
																	lowercase							=						normalizer_class(**A__)

												lowercase							=						do_lower_case




							def  A__      (				self   ,A__   ,A__=None):
												lowercase							=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

												return output




							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						[self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						self._tokenizer.model.save(A__   ,name=A__)
												return tuple(A__)





'''simple docstring'''




import unittest

import numpy as np

from transformers.file_utils import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import DPTImageProcessor






class 				a__(					unittest.TestCase	):


			'''simple docstring'''






			def __init__(   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase=7       ,    __lowerCAmelCase=3       ,    __lowerCAmelCase=18       ,    __lowerCAmelCase=30       ,    __lowerCAmelCase=400       ,    __lowerCAmelCase=True       ,    __lowerCAmelCase=None       ,    __lowerCAmelCase=True       ,    __lowerCAmelCase=[0.5, 0.5, 0.5]       ,    __lowerCAmelCase=[0.5, 0.5, 0.5]       ,    ):




					"""simple docstring"""


					lowerCAmelCase   					=  size if size is not None else {"""height""": 18, """width""": 18}
					lowerCAmelCase   					=  parent
					lowerCAmelCase   					=  batch_size
					lowerCAmelCase   					=  num_channels
					lowerCAmelCase   					=  image_size
					lowerCAmelCase   					=  min_resolution
					lowerCAmelCase   					=  max_resolution
					lowerCAmelCase   					=  do_resize
					lowerCAmelCase   					=  size
					lowerCAmelCase   					=  do_normalize
					lowerCAmelCase   					=  image_mean
					lowerCAmelCase   					=  image_std







			def 							a_       (   self):




					"""simple docstring"""


					return {
					    "image_mean": self.image_mean,
					    "image_std": self.image_std,
					    "do_normalize": self.do_normalize,
					    "do_resize": self.do_resize,
					    "size": self.size,
					}







@require_torch
@require_vision
class 				a__(					lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     Tuple      =       DPTImageProcessor if is_vision_available() else None



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  DPTImageProcessingTester(self)



			@property
			def 							a_       (   self):




					"""simple docstring"""


					return self.image_processor_tester.prepare_image_processor_dict()



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.image_processing_class(**self.image_processor_dict)
					self.assertTrue(hasattr(__lowerCAmelCase       ,    """image_mean"""))
					self.assertTrue(hasattr(__lowerCAmelCase       ,    """image_std"""))
					self.assertTrue(hasattr(__lowerCAmelCase       ,    """do_normalize"""))
					self.assertTrue(hasattr(__lowerCAmelCase       ,    """do_resize"""))
					self.assertTrue(hasattr(__lowerCAmelCase       ,    """size"""))



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.image_processing_class.from_dict(self.image_processor_dict)
					self.assertEqual(image_processor.size       ,    {"""height""": 18, """width""": 18})

					lowerCAmelCase   					=  self.image_processing_class.from_dict(self.image_processor_dict       ,    size=42)
					self.assertEqual(image_processor.size       ,    {"""height""": 42, """width""": 42})



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.image_processing_class(**self.image_processor_dict)
					# create random PIL images
					lowerCAmelCase   					=  prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__lowerCAmelCase)
					for image in image_inputs:
							self.assertIsInstance(__lowerCAmelCase       ,    Image.Image)

					# Test not batched input
					lowerCAmelCase   					=  image_processing(image_inputs[0]       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )

					# Test batched
					lowerCAmelCase   					=  image_processing(__lowerCAmelCase       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.image_processing_class(**self.image_processor_dict)
					# create random numpy tensors
					lowerCAmelCase   					=  prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__lowerCAmelCase       ,    numpify=__lowerCAmelCase)
					for image in image_inputs:
							self.assertIsInstance(__lowerCAmelCase       ,    np.ndarray)

					# Test not batched input
					lowerCAmelCase   					=  image_processing(image_inputs[0]       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )

					# Test batched
					lowerCAmelCase   					=  image_processing(__lowerCAmelCase       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )







			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.image_processing_class(**self.image_processor_dict)
					# create random PyTorch tensors
					lowerCAmelCase   					=  prepare_image_inputs(self.image_processor_tester       ,    equal_resolution=__lowerCAmelCase       ,    torchify=__lowerCAmelCase)
					for image in image_inputs:
							self.assertIsInstance(__lowerCAmelCase       ,    torch.Tensor)

					# Test not batched input
					lowerCAmelCase   					=  image_processing(image_inputs[0]       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )

					# Test batched
					lowerCAmelCase   					=  image_processing(__lowerCAmelCase       ,    return_tensors="""pt""").pixel_values
					self.assertEqual(
					    encoded_images.shape       ,    (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.size["""height"""],
					        self.image_processor_tester.size["""width"""],
					    )       ,    )




