






"""simple docstring"""





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE					:				Optional[Any]					    =						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE					:				str					    =						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_SCREAMING_SNAKE_CASE					:				Optional[Any]					    =						{
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

_SCREAMING_SNAKE_CASE					:				Tuple					    =						{
    'ctrl': 256,
}

_SCREAMING_SNAKE_CASE					:				Optional[Any]					    =						{
    'Pregnancy': 16_8629,
    'Christianity': 7675,
    'Explain': 10_6423,
    'Fitness': 6_3440,
    'Saving': 6_3163,
    'Ask': 2_7171,
    'Ass': 9_5985,
    'Joke': 16_3509,
    'Questions': 4_5622,
    'Thoughts': 4_9605,
    'Retail': 5_2342,
    'Feminism': 16_4338,
    'Writing': 1_1992,
    'Atheism': 19_2263,
    'Netflix': 4_8616,
    'Computing': 3_9639,
    'Opinion': 4_3213,
    'Alone': 4_4967,
    'Funny': 5_8917,
    'Gaming': 4_0358,
    'Human': 4088,
    'India': 1331,
    'Joker': 7_7138,
    'Diet': 3_6206,
    'Legal': 1_1859,
    'Norman': 4939,
    'Tip': 7_2689,
    'Weight': 5_2343,
    'Movies': 4_6273,
    'Running': 2_3425,
    'Science': 2090,
    'Horror': 3_7793,
    'Confession': 6_0572,
    'Finance': 1_2250,
    'Politics': 1_6360,
    'Scary': 19_1985,
    'Support': 1_2654,
    'Technologies': 3_2516,
    'Teenage': 6_6160,
    'Event': 3_2769,
    'Learned': 6_7460,
    'Notion': 18_2770,
    'Wikipedia': 3_7583,
    'Books': 6665,
    'Extract': 7_6050,
    'Confessions': 10_2701,
    'Conspiracy': 7_5932,
    'Links': 6_3674,
    'Narcissus': 15_0425,
    'Relationship': 5_4766,
    'Relationships': 13_4796,
    'Reviews': 4_1671,
    'News': 4256,
    'Translation': 2_6820,
    'multilingual': 12_8406,
}







def      lowerCamelCase__       (							_lowerCamelCase       :		Optional[int] )      ->				List[str]:
  lowerCamelCase_         =						set()
  lowerCamelCase_         =						word[0]
  for char in word[1:]:
    pairs.add((prev_char, char) )
    lowerCamelCase_         =						char

  lowerCamelCase_         =						set(_UpperCamelCase )
  return pairs






class      a						(					__snake_case		):
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						:     List[Any]						= PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE						:     List[Any]						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE						:     Union[str, Any]						= CONTROL_CODES






    def __init__(       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							Tuple="<unk>"       ,						**__SCREAMING_SNAKE_CASE     :							List[Any]					)     ->		Any:
      super().__init__(unk_token=__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE					)

      with open(__SCREAMING_SNAKE_CASE       ,						encoding='utf-8'					) as vocab_handle:
        lowerCamelCase_         =						json.load(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						{v: k for k, v in self.encoder.items()}
      with open(__SCREAMING_SNAKE_CASE       ,						encoding='utf-8'					) as merges_handle:
        lowerCamelCase_         =						merges_handle.read().split('\n'					)[1:-1]
      lowerCamelCase_         =						[tuple(merge.split()					) for merge in merges]
      lowerCamelCase_         =						dict(zip(__SCREAMING_SNAKE_CASE       ,						range(len(__SCREAMING_SNAKE_CASE					)					)					)					)
      lowerCamelCase_         =						{}






    @property
    def    UpperCamelCase       (       self     :							int					)     ->		str:
      return len(self.encoder					)






    def    UpperCamelCase       (       self     :							Optional[int]					)     ->		Tuple:
      return dict(self.encoder       ,						**self.added_tokens_encoder					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							int					)     ->		Tuple:






      if token in self.cache:
        return self.cache[token]
      lowerCamelCase_         =						tuple(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						tuple(list(word[:-1]					) + [word[-1] + '</w>']					)
      lowerCamelCase_         =						get_pairs(__SCREAMING_SNAKE_CASE					)

      if not pairs:
        return token

      while True:
        lowerCamelCase_         =						min(__SCREAMING_SNAKE_CASE       ,						key=lambda __SCREAMING_SNAKE_CASE					: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE       ,						float('inf'					)					)					)
        if bigram not in self.bpe_ranks:
          break
        lowerCamelCase_         =						bigram
        lowerCamelCase_         =						[]
        lowerCamelCase_         =						0
        while i < len(__SCREAMING_SNAKE_CASE					):
          try:
            lowerCamelCase_         =						word.index(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)
          except ValueError:
            new_word.extend(word[i:]					)
            break
          else:
            new_word.extend(word[i:j]					)
            lowerCamelCase_         =						j

          if word[i] == first and i < len(__SCREAMING_SNAKE_CASE					) - 1 and word[i + 1] == second:
            new_word.append(first + second					)
            i += 2
          else:
            new_word.append(word[i]					)
            i += 1
        lowerCamelCase_         =						tuple(__SCREAMING_SNAKE_CASE					)
        lowerCamelCase_         =						new_word
        if len(__SCREAMING_SNAKE_CASE					) == 1:
          break
        else:
          lowerCamelCase_         =						get_pairs(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						"""@@ """.join(__SCREAMING_SNAKE_CASE					)
      lowerCamelCase_         =						word[:-4]
      lowerCamelCase_         =						word
      return word






    def    UpperCamelCase       (       self     :							Optional[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]					)     ->		List[Any]:
      lowerCamelCase_         =						[]

      lowerCamelCase_         =						re.findall(r'\S+\n?'       ,						__SCREAMING_SNAKE_CASE					)

      for token in words:
        split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE					).split(' '					)					)					)
      return split_tokens






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[int]					)     ->		Dict:
      return self.encoder.get(__SCREAMING_SNAKE_CASE       ,						self.encoder.get(self.unk_token					)					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]					)     ->		List[Any]:
      return self.decoder.get(__SCREAMING_SNAKE_CASE       ,						self.unk_token					)






    def    UpperCamelCase       (       self     :							Optional[Any]       ,						__SCREAMING_SNAKE_CASE     :							List[str]					)     ->		Union[str, Any]:
      lowerCamelCase_         =						""" """.join(__SCREAMING_SNAKE_CASE					).replace('@@ '       ,						''					).strip()
      return out_string






    def    UpperCamelCase       (       self     :							Any       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[str] = None					)     ->		str:






      if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
        return
      lowerCamelCase_         =						os.path.join(
          __SCREAMING_SNAKE_CASE       ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
      lowerCamelCase_         =						os.path.join(
          __SCREAMING_SNAKE_CASE       ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

      with open(__SCREAMING_SNAKE_CASE       ,						'w'       ,						encoding='utf-8'					) as f:
        f.write(json.dumps(self.encoder       ,						indent=2       ,						sort_keys=__SCREAMING_SNAKE_CASE       ,						ensure_ascii=__SCREAMING_SNAKE_CASE					) + '\n'					)

      lowerCamelCase_         =						0
      with open(__SCREAMING_SNAKE_CASE       ,						'w'       ,						encoding='utf-8'					) as writer:
        writer.write('#version: 0.2\n'					)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,						key=lambda __SCREAMING_SNAKE_CASE					: kv[1]					):
          if index != token_index:
            logger.warning(
                F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                ' Please check that the tokenizer is not corrupted!'					)
            lowerCamelCase_         =						token_index
          writer.write(' '.join(__SCREAMING_SNAKE_CASE					) + '\n'					)
          index += 1

      return vocab_file, merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)








"""simple docstring"""






import unittest

from transformers import MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, is_vision_available, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
							from PIL import Image
else:





							class      lowerCamelCase__   :







												"""simple docstring"""


												@staticmethod
												def lowerCamelCase__							(       *UpperCamelCase    : List[str] ,       **UpperCamelCase    : Any  ):
																			'''simple docstring'''






																			pass





@is_pipeline_test
@require_vision
@require_torch
class      lowerCamelCase__   (     unittest.TestCase				):







					"""simple docstring"""


					__a	       =							MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : Tuple ,       UpperCamelCase    : Optional[int] ,       UpperCamelCase    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					pipeline(
												    """zero-shot-object-detection""" ,       model="""hf-internal-testing/tiny-random-owlvit-object-detection"""  )

												__UpperCAmelCase   :						Tuple      							=					[
												    {
												        """image""": """./tests/fixtures/tests_samples/COCO/000000039769.png""",
												        """candidate_labels""": ["""cat""", """remote""", """couch"""],
												    }
												]
												return object_detector, examples






					def lowerCamelCase__							(       self    : int ,       UpperCamelCase    : Optional[int] ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					object_detector(examples[0] ,       threshold=0.0  )

												__UpperCAmelCase   :						Tuple      							=					len(UpperCamelCase  )
												self.assertGreater(UpperCamelCase ,       0  )
												self.assertEqual(
												    UpperCamelCase ,       [
												        {
												            """score""": ANY(UpperCamelCase  ),
												            """label""": ANY(UpperCamelCase  ),
												            """box""": {"""xmin""": ANY(UpperCamelCase  ), """ymin""": ANY(UpperCamelCase  ), """xmax""": ANY(UpperCamelCase  ), """ymax""": ANY(UpperCamelCase  )},
												        }
												        for i in range(UpperCamelCase  )
												    ] ,       )






					@require_tf
					@unittest.skip("""Zero Shot Object Detection not implemented in TF"""  )
					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												pass






					@require_torch
					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					pipeline(
												    """zero-shot-object-detection""" ,       model="""hf-internal-testing/tiny-random-owlvit-object-detection"""  )

												__UpperCAmelCase   :						Tuple      							=					object_detector(
												    """./tests/fixtures/tests_samples/COCO/000000039769.png""" ,       candidate_labels=["""cat""", """remote""", """couch"""] ,       threshold=0.64 ,       )

												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        {"""score""": 0.7235, """label""": """cat""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												        {"""score""": 0.7218, """label""": """remote""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												        {"""score""": 0.7184, """label""": """couch""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												        {"""score""": 0.6748, """label""": """remote""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												        {"""score""": 0.6656, """label""": """cat""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												        {"""score""": 0.6614, """label""": """couch""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												        {"""score""": 0.6456, """label""": """remote""", """box""": {"""xmin""": 494, """ymin""": 105, """xmax""": 521, """ymax""": 127}},
												        {"""score""": 0.642, """label""": """remote""", """box""": {"""xmin""": 67, """ymin""": 274, """xmax""": 93, """ymax""": 297}},
												        {"""score""": 0.6419, """label""": """cat""", """box""": {"""xmin""": 494, """ymin""": 105, """xmax""": 521, """ymax""": 127}},
												    ] ,       )

												__UpperCAmelCase   :						Any      							=					object_detector(
												    [
												        {
												            """image""": """./tests/fixtures/tests_samples/COCO/000000039769.png""",
												            """candidate_labels""": ["""cat""", """remote""", """couch"""],
												        }
												    ] ,       threshold=0.64 ,       )

												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        [
												            {"""score""": 0.7235, """label""": """cat""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												            {"""score""": 0.7218, """label""": """remote""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												            {"""score""": 0.7184, """label""": """couch""", """box""": {"""xmin""": 204, """ymin""": 167, """xmax""": 232, """ymax""": 190}},
												            {"""score""": 0.6748, """label""": """remote""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												            {"""score""": 0.6656, """label""": """cat""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												            {"""score""": 0.6614, """label""": """couch""", """box""": {"""xmin""": 571, """ymin""": 83, """xmax""": 598, """ymax""": 103}},
												            {"""score""": 0.6456, """label""": """remote""", """box""": {"""xmin""": 494, """ymin""": 105, """xmax""": 521, """ymax""": 127}},
												            {"""score""": 0.642, """label""": """remote""", """box""": {"""xmin""": 67, """ymin""": 274, """xmax""": 93, """ymax""": 297}},
												            {"""score""": 0.6419, """label""": """cat""", """box""": {"""xmin""": 494, """ymin""": 105, """xmax""": 521, """ymax""": 127}},
												        ]
												    ] ,       )






					@require_torch
					@slow
					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					pipeline("""zero-shot-object-detection"""  )

												__UpperCAmelCase   :						int      							=					object_detector(
												    """http://images.cocodataset.org/val2017/000000039769.jpg""" ,       candidate_labels=["""cat""", """remote""", """couch"""] ,       )
												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        {"""score""": 0.2868, """label""": """cat""", """box""": {"""xmin""": 324, """ymin""": 20, """xmax""": 640, """ymax""": 373}},
												        {"""score""": 0.277, """label""": """remote""", """box""": {"""xmin""": 40, """ymin""": 72, """xmax""": 177, """ymax""": 115}},
												        {"""score""": 0.2537, """label""": """cat""", """box""": {"""xmin""": 1, """ymin""": 55, """xmax""": 315, """ymax""": 472}},
												        {"""score""": 0.1474, """label""": """remote""", """box""": {"""xmin""": 335, """ymin""": 74, """xmax""": 371, """ymax""": 187}},
												        {"""score""": 0.1208, """label""": """couch""", """box""": {"""xmin""": 4, """ymin""": 0, """xmax""": 642, """ymax""": 476}},
												    ] ,       )

												__UpperCAmelCase   :						List[str]      							=					object_detector(
												    [
												        {
												            """image""": """http://images.cocodataset.org/val2017/000000039769.jpg""",
												            """candidate_labels""": ["""cat""", """remote""", """couch"""],
												        },
												        {
												            """image""": """http://images.cocodataset.org/val2017/000000039769.jpg""",
												            """candidate_labels""": ["""cat""", """remote""", """couch"""],
												        },
												    ] ,       )
												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        [
												            {"""score""": 0.2868, """label""": """cat""", """box""": {"""xmin""": 324, """ymin""": 20, """xmax""": 640, """ymax""": 373}},
												            {"""score""": 0.277, """label""": """remote""", """box""": {"""xmin""": 40, """ymin""": 72, """xmax""": 177, """ymax""": 115}},
												            {"""score""": 0.2537, """label""": """cat""", """box""": {"""xmin""": 1, """ymin""": 55, """xmax""": 315, """ymax""": 472}},
												            {"""score""": 0.1474, """label""": """remote""", """box""": {"""xmin""": 335, """ymin""": 74, """xmax""": 371, """ymax""": 187}},
												            {"""score""": 0.1208, """label""": """couch""", """box""": {"""xmin""": 4, """ymin""": 0, """xmax""": 642, """ymax""": 476}},
												        ],
												        [
												            {"""score""": 0.2868, """label""": """cat""", """box""": {"""xmin""": 324, """ymin""": 20, """xmax""": 640, """ymax""": 373}},
												            {"""score""": 0.277, """label""": """remote""", """box""": {"""xmin""": 40, """ymin""": 72, """xmax""": 177, """ymax""": 115}},
												            {"""score""": 0.2537, """label""": """cat""", """box""": {"""xmin""": 1, """ymin""": 55, """xmax""": 315, """ymax""": 472}},
												            {"""score""": 0.1474, """label""": """remote""", """box""": {"""xmin""": 335, """ymin""": 74, """xmax""": 371, """ymax""": 187}},
												            {"""score""": 0.1208, """label""": """couch""", """box""": {"""xmin""": 4, """ymin""": 0, """xmax""": 642, """ymax""": 476}},
												        ],
												    ] ,       )






					@require_tf
					@unittest.skip("""Zero Shot Object Detection not implemented in TF"""  )
					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												pass






					@require_torch
					@slow
					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					0.2
												__UpperCAmelCase   :						List[Any]      							=					pipeline("""zero-shot-object-detection"""  )

												__UpperCAmelCase   :						Any      							=					object_detector(
												    """http://images.cocodataset.org/val2017/000000039769.jpg""" ,       candidate_labels=["""cat""", """remote""", """couch"""] ,       threshold=UpperCamelCase ,       )
												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        {"""score""": 0.2868, """label""": """cat""", """box""": {"""xmin""": 324, """ymin""": 20, """xmax""": 640, """ymax""": 373}},
												        {"""score""": 0.277, """label""": """remote""", """box""": {"""xmin""": 40, """ymin""": 72, """xmax""": 177, """ymax""": 115}},
												        {"""score""": 0.2537, """label""": """cat""", """box""": {"""xmin""": 1, """ymin""": 55, """xmax""": 315, """ymax""": 472}},
												    ] ,       )






					@require_torch
					@slow
					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					2
												__UpperCAmelCase   :						Union[str, Any]      							=					pipeline("""zero-shot-object-detection"""  )

												__UpperCAmelCase   :						List[str]      							=					object_detector(
												    """http://images.cocodataset.org/val2017/000000039769.jpg""" ,       candidate_labels=["""cat""", """remote""", """couch"""] ,       top_k=UpperCamelCase ,       )
												self.assertEqual(
												    nested_simplify(UpperCamelCase ,       decimals=4  ) ,       [
												        {"""score""": 0.2868, """label""": """cat""", """box""": {"""xmin""": 324, """ymin""": 20, """xmax""": 640, """ymax""": 373}},
												        {"""score""": 0.277, """label""": """remote""", """box""": {"""xmin""": 40, """ymin""": 72, """xmax""": 177, """ymax""": 115}},
												    ] ,       )
