

import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
					from PIL import Image

					from transformers import OwlViTImageProcessor, OwlViTProcessor







@require_vision
class  lowercase				(       unittest.TestCase			):




				def     a							(					self       ):
											snake_case_							=    tempfile.mkdtemp()

											# fmt: off
											snake_case_							=    ['', 'l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
											# fmt: on
											snake_case_							=    dict(zip(snake_case      ,   range(len(snake_case       )       )       )       )
											snake_case_							=    ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
											snake_case_							=    {'unk_token': '<unk>'}

											snake_case_							=    os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']       )
											snake_case_							=    os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		fp.write(json.dumps(snake_case       ) + '\n'       )
											with open(self.merges_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		fp.write('\n'.join(snake_case       )       )

											snake_case_							=    {
											    'do_resize': True,
											    'size': 20,
											    'do_center_crop': True,
											    'crop_size': 18,
											    'do_normalize': True,
											    'image_mean': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
											    'image_std': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
											}
											snake_case_							=    os.path.join(self.tmpdirname      ,   snake_case       )
											with open(self.image_processor_file      ,   'w'      ,   encoding='utf-8'       ) as fp:
																		json.dump(snake_case      ,   snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return CLIPTokenizer.from_pretrained(self.tmpdirname      ,   pad_token='!'      ,   **snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname      ,   pad_token='!'      ,   **snake_case       )




				def     a							(					self      ,   **snake_case       ):
											return OwlViTImageProcessor.from_pretrained(self.tmpdirname      ,   **snake_case       )




				def     a							(					self       ):
											shutil.rmtree(self.tmpdirname       )




				def     a							(					self       ):
											snake_case_							=    [np.random.randint(255      ,   size=(3, 30, 400)      ,   dtype=np.uinta       )]

											snake_case_							=    [Image.fromarray(np.moveaxis(snake_case      ,   0      ,   -1       )       ) for x in image_inputs]

											return image_inputs




				def     a							(					self       ):
											snake_case_							=    self.get_tokenizer()
											snake_case_							=    self.get_rust_tokenizer()
											snake_case_							=    self.get_image_processor()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )
											processor_slow.save_pretrained(self.tmpdirname       )
											snake_case_							=    OwlViTProcessor.from_pretrained(self.tmpdirname      ,   use_fast=snake_case       )

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )
											processor_fast.save_pretrained(self.tmpdirname       )
											snake_case_							=    OwlViTProcessor.from_pretrained(self.tmpdirname       )

											self.assertEqual(processor_slow.tokenizer.get_vocab()      ,   tokenizer_slow.get_vocab()       )
											self.assertEqual(processor_fast.tokenizer.get_vocab()      ,   tokenizer_fast.get_vocab()       )
											self.assertEqual(tokenizer_slow.get_vocab()      ,   tokenizer_fast.get_vocab()       )
											self.assertIsInstance(processor_slow.tokenizer      ,   snake_case       )
											self.assertIsInstance(processor_fast.tokenizer      ,   snake_case       )

											self.assertEqual(processor_slow.image_processor.to_json_string()      ,   image_processor.to_json_string()       )
											self.assertEqual(processor_fast.image_processor.to_json_string()      ,   image_processor.to_json_string()       )
											self.assertIsInstance(processor_slow.image_processor      ,   snake_case       )
											self.assertIsInstance(processor_fast.image_processor      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    OwlViTProcessor(tokenizer=self.get_tokenizer()      ,   image_processor=self.get_image_processor()       )
											processor.save_pretrained(self.tmpdirname       )

											snake_case_							=    self.get_tokenizer(bos_token='(BOS)'      ,   eos_token='(EOS)'       )
											snake_case_							=    self.get_image_processor(do_normalize=snake_case       )

											snake_case_							=    OwlViTProcessor.from_pretrained(
											    self.tmpdirname      ,   bos_token='(BOS)'      ,   eos_token='(EOS)'      ,   do_normalize=snake_case       )

											self.assertEqual(processor.tokenizer.get_vocab()      ,   tokenizer_add_kwargs.get_vocab()       )
											self.assertIsInstance(processor.tokenizer      ,   snake_case       )

											self.assertEqual(processor.image_processor.to_json_string()      ,   image_processor_add_kwargs.to_json_string()       )
											self.assertIsInstance(processor.image_processor      ,   snake_case       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    image_processor(snake_case      ,   return_tensors='np'       )
											snake_case_							=    processor(images=snake_case      ,   return_tensors='np'       )

											for key in input_image_proc.keys():
																		self.assertAlmostEqual(input_image_proc[key].sum()      ,   input_processor[key].sum()      ,   delta=1e-2       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    'lower newer'

											snake_case_							=    processor(text=snake_case      ,   return_tensors='np'       )

											snake_case_							=    tokenizer(snake_case      ,   return_tensors='np'       )

											for key in encoded_tok.keys():
																		self.assertListEqual(encoded_tok[key][0].tolist()      ,   encoded_processor[key][0].tolist()       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    'lower newer'
											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    processor(text=snake_case      ,   images=snake_case       )

											self.assertListEqual(list(inputs.keys()       )      ,   ['input_ids', 'attention_mask', 'pixel_values']       )

											# test if it raises when no input is passed
											with pytest.raises(snake_case       ):
																		processor()




				def     a							(					self       ):
											snake_case_							=    'google/owlvit-base-patch32'
											snake_case_							=    OwlViTProcessor.from_pretrained(snake_case       )

											snake_case_							=    ['cat', 'nasa badge']
											snake_case_							=    processor(text=snake_case       )

											snake_case_							=    16
											self.assertListEqual(list(inputs.keys()       )      ,   ['input_ids', 'attention_mask']       )
											self.assertEqual(inputs['input_ids'].shape      ,   (2, seq_length)       )

											# test if it raises when no input is passed
											with pytest.raises(snake_case       ):
																		processor()




				def     a							(					self       ):
											snake_case_							=    'google/owlvit-base-patch32'
											snake_case_							=    OwlViTProcessor.from_pretrained(snake_case       )

											snake_case_							=    [['cat', 'nasa badge'], ['person']]
											snake_case_							=    processor(text=snake_case       )

											snake_case_							=    16
											snake_case_							=    len(snake_case       )
											snake_case_							=    max([len(snake_case       ) for texts in input_texts]       )

											self.assertListEqual(list(inputs.keys()       )      ,   ['input_ids', 'attention_mask']       )
											self.assertEqual(inputs['input_ids'].shape      ,   (batch_size * num_max_text_queries, seq_length)       )

											# test if it raises when no input is passed
											with pytest.raises(snake_case       ):
																		processor()




				def     a							(					self       ):
											snake_case_							=    'google/owlvit-base-patch32'
											snake_case_							=    OwlViTProcessor.from_pretrained(snake_case       )

											snake_case_							=    ['cat', 'nasa badge']
											snake_case_							=    processor(text=snake_case       )

											snake_case_							=    16
											snake_case_							=    inputs['input_ids']
											snake_case_							=    [
											    [4_9406, 2368, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											    [4_9406, 6841, 1_1301, 4_9407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											]

											self.assertListEqual(list(inputs.keys()       )      ,   ['input_ids', 'attention_mask']       )
											self.assertEqual(inputs['input_ids'].shape      ,   (2, seq_length)       )
											self.assertListEqual(list(input_ids[0]       )      ,   predicted_ids[0]       )
											self.assertListEqual(list(input_ids[1]       )      ,   predicted_ids[1]       )




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    self.prepare_image_inputs()
											snake_case_							=    self.prepare_image_inputs()

											snake_case_							=    processor(images=snake_case      ,   query_images=snake_case       )

											self.assertListEqual(list(inputs.keys()       )      ,   ['query_pixel_values', 'pixel_values']       )

											# test if it raises when no input is passed
											with pytest.raises(snake_case       ):
																		processor()




				def     a							(					self       ):
											snake_case_							=    self.get_image_processor()
											snake_case_							=    self.get_tokenizer()

											snake_case_							=    OwlViTProcessor(tokenizer=snake_case      ,   image_processor=snake_case       )

											snake_case_							=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											snake_case_							=    processor.batch_decode(snake_case       )
											snake_case_							=    tokenizer.batch_decode(snake_case       )

											self.assertListEqual(snake_case      ,   snake_case       )







from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase		:   Any										=       logging.get_logger(__name__)


_UpperCAmelCase		:   Dict										=       {
    """nielsr/canine-s""": 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_UpperCAmelCase		:   Tuple										=       111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_UpperCAmelCase		:   List[str]										=       0

_UpperCAmelCase		:   Any										=       0xE000
_UpperCAmelCase		:   Dict										=       0xE001
_UpperCAmelCase		:   Optional[int]										=       0xE002
_UpperCAmelCase		:   Tuple										=       0xE003
_UpperCAmelCase		:   Tuple										=       0xE004

# Maps special codepoints to human-readable names.
_UpperCAmelCase		:   Dict[int, str]										=       {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_UpperCAmelCase		:   Dict[str, int]										=       {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}







class  lowercase				(       lowercase_			):
				__SCREAMING_SNAKE_CASE					: str       			=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




				def __init__(					self      ,   snake_case=chr(snake_case       )      ,   snake_case=chr(snake_case       )      ,   snake_case=chr(snake_case       )      ,   snake_case=chr(snake_case       )      ,   snake_case=chr(snake_case       )      ,   snake_case=chr(snake_case       )      ,   snake_case=False      ,   snake_case=2048      ,   **snake_case      ,   ):
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else bos_token
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else eos_token
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else sep_token
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else cls_token
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											snake_case_							=    AddedToken(snake_case      ,   lstrip=snake_case      ,   rstrip=snake_case       ) if isinstance(snake_case      ,   snake_case       ) else mask_token

											super().__init__(
											    bos_token=snake_case      ,   eos_token=snake_case      ,   sep_token=snake_case      ,   cls_token=snake_case      ,   pad_token=snake_case      ,   mask_token=snake_case      ,   add_prefix_space=snake_case      ,   model_max_length=snake_case      ,   **snake_case      ,   )

											# Creates a mapping for looking up the IDs of special symbols.
											snake_case_							=    {}
											for codepoint, name in SPECIAL_CODEPOINTS.items():
																		snake_case_							=    codepoint

											# Creates a mapping for looking up the string forms of special symbol IDs.
											snake_case_							=    {
											    codepoint: name for name, codepoint in self._special_codepoints.items()
											}

											snake_case_							=    UNICODE_VOCAB_SIZE
											snake_case_							=    len(self._special_codepoints       )




				@property
				def     a							(					self       ):
											return self._unicode_vocab_size




				def     a							(					self      ,   snake_case       ):
											return list(snake_case       )




				def     a							(					self      ,   snake_case       ):
											try:
																		return ord(snake_case       )
											except TypeError:
																		raise ValueError(F'''invalid token: \'{token}\''''       )




				def     a							(					self      ,   snake_case       ):
											try:
																		if index in SPECIAL_CODEPOINTS:
																									return SPECIAL_CODEPOINTS[index]
																		return chr(snake_case       )
											except TypeError:
																		raise ValueError(F'''invalid id: {index}'''       )




				def     a							(					self      ,   snake_case       ):
											return "".join(snake_case       )




				def     a							(					self      ,   snake_case      ,   snake_case = None       ):
											snake_case_							=    [self.sep_token_id]
											snake_case_							=    [self.cls_token_id]

											snake_case_							=    cls + token_ids_a + sep
											if token_ids_a is not None:
																		result += token_ids_a + sep
											return result




				def     a							(					self      ,   snake_case      ,   snake_case = None      ,   snake_case = False       ):
											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=snake_case      ,   token_ids_a=snake_case      ,   already_has_special_tokens=snake_case       )

											snake_case_							=    [1] + ([0] * len(snake_case       )) + [1]
											if token_ids_a is not None:
																		result += ([0] * len(snake_case       )) + [1]
											return result




				def     a							(					self      ,   snake_case      ,   snake_case = None       ):
											snake_case_							=    [self.sep_token_id]
											snake_case_							=    [self.cls_token_id]

											snake_case_							=    len(cls + token_ids_a + sep       ) * [0]
											if token_ids_a is not None:
																		result += len(token_ids_a + sep       ) * [1]
											return result




				def     a							(					self      ,   snake_case      ,   snake_case = None       ):
											return ()




