
'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__lowercase							:      List[str]		 =   logging.get_logger(__name__)

__lowercase							:      List[str]		 =   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__lowercase							:      List[Any]		 =   {
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

__lowercase							:      List[str]		 =   {
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

__lowercase							:      Dict		 =   {
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}

class 		__lowercase (     _lowerCAmelCase			):
			lowerCamelCase	:						Optional[int]							=	VOCAB_FILES_NAMES
			lowerCamelCase	:						List[str]							=	PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase	:						Tuple							=	PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase	:						List[Any]							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase	:						str							=	RealmTokenizer

			def __init__(self	,      A=None	,      A=None	,      A=True	,      A="[UNK]"	,      A="[SEP]"	,      A="[PAD]"	,      A="[CLS]"	,      A="[MASK]"	,      A=True	,      A=None	,      **A	,      ):
								super().__init__(
								    _lowerCamelCase	,      tokenizer_file=_lowerCamelCase	,      do_lower_case=_lowerCamelCase	,      unk_token=_lowerCamelCase	,      sep_token=_lowerCamelCase	,      pad_token=_lowerCamelCase	,      cls_token=_lowerCamelCase	,      mask_token=_lowerCamelCase	,      tokenize_chinese_chars=_lowerCamelCase	,      strip_accents=_lowerCamelCase	,      **_lowerCamelCase	,      )

								lowerCamelCase_    :					Any   =						json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
								if (
								    normalizer_state.get('''lowercase'''	,      _lowerCamelCase					) != do_lower_case
								    or normalizer_state.get('''strip_accents'''	,      _lowerCamelCase					) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars'''	,      _lowerCamelCase					) != tokenize_chinese_chars
								):
													lowerCamelCase_    :					List[Any]   =						getattr(_lowerCamelCase	,      normalizer_state.pop('''type'''					)					)
													lowerCamelCase_    :					Dict   =						do_lower_case
													lowerCamelCase_    :					Tuple   =						strip_accents
													lowerCamelCase_    :					int   =						tokenize_chinese_chars
													lowerCamelCase_    :					Union[str, Any]   =						normalizer_class(**_lowerCamelCase					)

								lowerCamelCase_    :					Dict   =						do_lower_case

			def 				UpperCAmelCase__       (self	,      A	,      **A					):
								lowerCamelCase_    :					Union[str, Any]   =						PaddingStrategy.MAX_LENGTH

								lowerCamelCase_    :					Union[str, Any]   =						text
								lowerCamelCase_    :					Union[str, Any]   =						kwargs.pop('''text_pair'''	,      _lowerCamelCase					)
								lowerCamelCase_    :					Optional[Any]   =						kwargs.pop('''return_tensors'''	,      _lowerCamelCase					)

								lowerCamelCase_    :					Any   =						{
								    '''input_ids''': [],
								    '''attention_mask''': [],
								    '''token_type_ids''': [],
								}

								for idx, candidate_text in enumerate(_lowerCamelCase					):
													if batch_text_pair is not None:
																		lowerCamelCase_    :					Dict   =						batch_text_pair[idx]
													else:
																		lowerCamelCase_    :					Optional[Any]   =						None

													lowerCamelCase_    :					str   =						super().__call__(_lowerCamelCase	,      _lowerCamelCase	,      return_tensors=_lowerCamelCase	,      **_lowerCamelCase					)

													lowerCamelCase_    :					Optional[Any]   =						encoded_candidates.get('''input_ids'''					)
													lowerCamelCase_    :					List[str]   =						encoded_candidates.get('''attention_mask'''					)
													lowerCamelCase_    :					Optional[int]   =						encoded_candidates.get('''token_type_ids'''					)

													if encoded_input_ids is not None:
																		output_data["input_ids"].append(_lowerCamelCase					)
													if encoded_attention_mask is not None:
																		output_data["attention_mask"].append(_lowerCamelCase					)
													if encoded_token_type_ids is not None:
																		output_data["token_type_ids"].append(_lowerCamelCase					)

								lowerCamelCase_    :					Union[str, Any]   =						{key: item for key, item in output_data.items() if len(_lowerCamelCase					) != 0}

								return BatchEncoding(_lowerCamelCase	,      tensor_type=_lowerCamelCase					)

			def 				UpperCAmelCase__       (self	,      A	,      A=None					):
								lowerCamelCase_    :					Any   =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output

			def 				UpperCAmelCase__       (self	,      A	,      A = None					):
								lowerCamelCase_    :					Tuple   =						[self.sep_token_id]
								lowerCamelCase_    :					int   =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






			def 				UpperCAmelCase__       (self	,      A	,      A = None					):
								lowerCamelCase_    :					int   =						self._tokenizer.model.save(_lowerCamelCase	,      name=_lowerCamelCase					)
								return tuple(_lowerCamelCase					)










"""simple docstring"""


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
       import torch

if is_vision_available():
       from PIL import Image

       from transformers import ViTImageProcessor


class 					snake_case		(							unittest.TestCase						):


   '''simple docstring'''

   def __init__(	self :     List[str],       _lowerCamelCase :     Optional[Any],       _lowerCamelCase :     Union[str, Any]=13,       _lowerCamelCase :     Any=3,       _lowerCamelCase :     Optional[int]=2_24,       _lowerCamelCase :     str=30,       _lowerCamelCase :     Dict=4_00,       _lowerCamelCase :     Union[str, Any]=True,       _lowerCamelCase :     Any=None,       _lowerCamelCase :     Optional[Any]=True,       _lowerCamelCase :     Any=[0.5, 0.5, 0.5],       _lowerCamelCase :     List[str]=[0.5, 0.5, 0.5],       ):

     '''simple docstring'''
     __A									= size if size is not None else {'''height''': 18, '''width''': 18}
     __A									= parent
     __A									= batch_size
     __A									= num_channels
     __A									= image_size
     __A									= min_resolution
     __A									= max_resolution
     __A									= do_resize
     __A									= size
     __A									= do_normalize
     __A									= image_mean
     __A									= image_std






   def 				_SCREAMING_SNAKE_CASE				(	self :     int  ):

     '''simple docstring'''
     return {
         "image_mean": self.image_mean,
         "image_std": self.image_std,
         "do_normalize": self.do_normalize,
         "do_resize": self.do_resize,
         "size": self.size,
     }





@require_torch
@require_vision
class 					snake_case		(							_lowerCAmelCase   ,	unittest.TestCase						):


   '''simple docstring'''

   A_						:	str						 = ViTImageProcessor if is_vision_available() else None


   def 				_SCREAMING_SNAKE_CASE				(	self :     Optional[int]  ):

     '''simple docstring'''
     __A									= EfficientFormerImageProcessorTester(self  )


   @property
   def 				_SCREAMING_SNAKE_CASE				(	self :     Dict  ):

     '''simple docstring'''
     return self.image_proc_tester.prepare_image_processor_dict()


   def 				_SCREAMING_SNAKE_CASE				(	self :     Optional[Any]  ):

     '''simple docstring'''
     __A									= self.image_processing_class(**self.image_processor_dict  )
     self.assertTrue(hasattr(_lowerCamelCase,       '''image_mean'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,       '''image_std'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,       '''do_normalize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,       '''do_resize'''  )  )
     self.assertTrue(hasattr(_lowerCamelCase,       '''size'''  )  )


   def 				_SCREAMING_SNAKE_CASE				(	self :     Union[str, Any]  ):

     '''simple docstring'''
     pass


   def 				_SCREAMING_SNAKE_CASE				(	self :     List[Any]  ):

     '''simple docstring'''
     # Initialize image_processor
     __A									= self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A									= prepare_image_inputs(self.image_proc_tester,       equal_resolution=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,       Image.Image  )

     # Test not batched input
     __A									= image_processor(image_inputs[0],       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             1,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )

     # Test batched
     __A									= image_processor(_lowerCamelCase,       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             self.image_proc_tester.batch_size,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )


   def 				_SCREAMING_SNAKE_CASE				(	self :     Tuple  ):

     '''simple docstring'''
     # Initialize image_processor
     __A									= self.image_processing_class(**self.image_processor_dict  )
     # create random numpy tensors
     __A									= prepare_image_inputs(self.image_proc_tester,       equal_resolution=_lowerCamelCase,       numpify=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,       np.ndarray  )

     # Test not batched input
     __A									= image_processor(image_inputs[0],       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             1,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )

     # Test batched
     __A									= image_processor(_lowerCamelCase,       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             self.image_proc_tester.batch_size,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )






   def 				_SCREAMING_SNAKE_CASE				(	self :     str  ):

     '''simple docstring'''
     # Initialize image_processor
     __A									= self.image_processing_class(**self.image_processor_dict  )
     # create random PyTorch tensors
     __A									= prepare_image_inputs(self.image_proc_tester,       equal_resolution=_lowerCamelCase,       torchify=_lowerCamelCase  )
     for image in image_inputs:
       self.assertIsInstance(_lowerCamelCase,       torch.Tensor  )

     # Test not batched input
     __A									= image_processor(image_inputs[0],       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             1,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )

     # Test batched
     __A									= image_processor(_lowerCamelCase,       return_tensors='''pt'''  ).pixel_values
     self.assertEqual(
         encoded_images.shape,       (
             self.image_proc_tester.batch_size,
             self.image_proc_tester.num_channels,
             self.image_proc_tester.size['''height'''],
             self.image_proc_tester.size['''width'''],
         ),       )




