


"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast
class     _UpperCamelCase    (  unittest.TestCase						):
							'''simple docstring'''







							def      snake_case							(	self      ):
										# A mock response for an HTTP head request to emulate server down
										__lowerCAmelCase							       =     mock.Mock()
										__lowerCAmelCase							       =     5_00
										__lowerCAmelCase							       =     {}
										__lowerCAmelCase							       =     HTTPError
										__lowerCAmelCase							       =     {}

										# Download this model to make sure it's in the cache.
										__lowerCAmelCase							       =     BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"       ,  return_value=__a      ) as mock_head:
													__lowerCAmelCase							       =     BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"      )
													# This check we did call the fake head request
													mock_head.assert_called()

							@require_tokenizers
							def      snake_case							(	self      ):
										# A mock response for an HTTP head request to emulate server down
										__lowerCAmelCase							       =     mock.Mock()
										__lowerCAmelCase							       =     5_00
										__lowerCAmelCase							       =     {}
										__lowerCAmelCase							       =     HTTPError
										__lowerCAmelCase							       =     {}

										# Download this model to make sure it's in the cache.
										__lowerCAmelCase							       =     GPTaTokenizerFast.from_pretrained("gpt2"      )

										# Under the mock environment we get a 500 error when trying to reach the tokenizer.
										with mock.patch("requests.Session.request"       ,  return_value=__a      ) as mock_head:
													__lowerCAmelCase							       =     GPTaTokenizerFast.from_pretrained("gpt2"      )
													# This check we did call the fake head request
													mock_head.assert_called()

							def      snake_case							(	self      ):
										# This test is for deprecated behavior and can be removed in v5
										try:
													__lowerCAmelCase							       =     tempfile.mktemp()
													with open(__a       ,  "wb"      ) as f:
																http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       ,  __a      )

													__lowerCAmelCase							       =     AlbertTokenizer.from_pretrained(__a      )
										finally:
													os.remove(__a      )

										# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
										# the current folder and have the right name.
										if os.path.isfile("tokenizer.json"      ):
													# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
													return
										try:
													with open("tokenizer.json"       ,  "wb"      ) as f:
																http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"       ,  __a      )
													__lowerCAmelCase							       =     AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"      )
													# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
													self.assertEqual(tokenizer.vocab_size       ,  10_00      )
													# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

										finally:
													os.remove("tokenizer.json"      )



							def      snake_case							(	self      ):
										# This test is for deprecated behavior and can be removed in v5
										__lowerCAmelCase							       =     AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"      )
@is_staging_test
class     _UpperCamelCase    (  unittest.TestCase						):
							'''simple docstring'''







							__UpperCAmelCase		:       int							 =["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

							@classmethod
							def      snake_case							(	cls      ):
										__lowerCAmelCase							       =     TOKEN
										HfFolder.save_token(__a      )

							@classmethod
							def      snake_case							(	cls      ):
										try:
													delete_repo(token=cls._token       ,  repo_id="test-tokenizer"      )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token       ,  repo_id="valid_org/test-tokenizer-org"      )
										except HTTPError:
													pass

										try:
													delete_repo(token=cls._token       ,  repo_id="test-dynamic-tokenizer"      )
										except HTTPError:
													pass

							def      snake_case							(	self      ):
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowerCAmelCase							       =     os.path.join(__a       ,  "vocab.txt"      )
													with open(__a       ,  "w"       ,  encoding="utf-8"      ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
													__lowerCAmelCase							       =     BertTokenizer(__a      )

										tokenizer.push_to_hub("test-tokenizer"       ,  use_auth_token=self._token      )
										__lowerCAmelCase							       =     BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"      )
										self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab      )

										# Reset repo
										delete_repo(token=self._token       ,  repo_id="test-tokenizer"      )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(__a       ,  repo_id="test-tokenizer"       ,  push_to_hub=__a       ,  use_auth_token=self._token      )

										__lowerCAmelCase							       =     BertTokenizer.from_pretrained(f"{USER}/test-tokenizer"      )
										self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab      )

							def      snake_case							(	self      ):
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowerCAmelCase							       =     os.path.join(__a       ,  "vocab.txt"      )
													with open(__a       ,  "w"       ,  encoding="utf-8"      ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
													__lowerCAmelCase							       =     BertTokenizer(__a      )

										tokenizer.push_to_hub("valid_org/test-tokenizer-org"       ,  use_auth_token=self._token      )
										__lowerCAmelCase							       =     BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
										self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab      )

										# Reset repo
										delete_repo(token=self._token       ,  repo_id="valid_org/test-tokenizer-org"      )

										# Push to hub via save_pretrained
										with tempfile.TemporaryDirectory() as tmp_dir:
													tokenizer.save_pretrained(
													    __a       ,  repo_id="valid_org/test-tokenizer-org"       ,  push_to_hub=__a       ,  use_auth_token=self._token      )

										__lowerCAmelCase							       =     BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"      )
										self.assertDictEqual(new_tokenizer.vocab       ,  tokenizer.vocab      )



							@require_tokenizers
							def      snake_case							(	self      ):
										CustomTokenizer.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowerCAmelCase							       =     os.path.join(__a       ,  "vocab.txt"      )
													with open(__a       ,  "w"       ,  encoding="utf-8"      ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )
													__lowerCAmelCase							       =     CustomTokenizer(__a      )

										# No fast custom tokenizer
										tokenizer.push_to_hub("test-dynamic-tokenizer"       ,  use_auth_token=self._token      )

										__lowerCAmelCase							       =     AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"       ,  trust_remote_code=__a      )
										# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__       ,  "CustomTokenizer"      )

										# Fast and slow custom tokenizer
										CustomTokenizerFast.register_for_auto_class()
										with tempfile.TemporaryDirectory() as tmp_dir:
													__lowerCAmelCase							       =     os.path.join(__a       ,  "vocab.txt"      )
													with open(__a       ,  "w"       ,  encoding="utf-8"      ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]      )      )

													__lowerCAmelCase							       =     BertTokenizerFast.from_pretrained(__a      )
													bert_tokenizer.save_pretrained(__a      )
													__lowerCAmelCase							       =     CustomTokenizerFast.from_pretrained(__a      )

										tokenizer.push_to_hub("test-dynamic-tokenizer"       ,  use_auth_token=self._token      )

										__lowerCAmelCase							       =     AutoTokenizer.from_pretrained(f"{USER}/test-dynamic-tokenizer"       ,  trust_remote_code=__a      )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__       ,  "CustomTokenizerFast"      )
										__lowerCAmelCase							       =     AutoTokenizer.from_pretrained(
										    f"{USER}/test-dynamic-tokenizer"       ,  use_fast=__a       ,  trust_remote_code=__a      )
										# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
										self.assertEqual(tokenizer.__class__.__name__       ,  "CustomTokenizer"      )
class     _UpperCamelCase    (  unittest.TestCase						):
							'''simple docstring'''







							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("Hello 友達"      )
										self.assertEqual(trie.data       ,  {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}      )
										trie.add("Hello"      )
										trie.data
										self.assertEqual(trie.data       ,  {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )       ,  ["[CLS] This is a extra_id_100"]      )
										trie.add("[CLS]"      )
										trie.add("extra_id_1"      )
										trie.add("extra_id_100"      )
										self.assertEqual(trie.split("[CLS] This is a extra_id_100"      )       ,  ["[CLS]", " This is a ", "extra_id_100"]      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("A"      )
										self.assertEqual(trie.split("ABC"      )       ,  ["A", "BC"]      )
										self.assertEqual(trie.split("BCA"      )       ,  ["BC", "A"]      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("TOKEN]"      )
										trie.add("[SPECIAL_TOKEN]"      )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )       ,  ["This is something ", "[SPECIAL_TOKEN]"]      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("A"      )
										trie.add("P"      )
										trie.add("[SPECIAL_TOKEN]"      )
										self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"      )       ,  ["This is something ", "[SPECIAL_TOKEN]"]      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("AB"      )
										trie.add("B"      )
										trie.add("C"      )
										self.assertEqual(trie.split("ABC"      )       ,  ["AB", "C"]      )

							def      snake_case							(	self      ):
										__lowerCAmelCase							       =     Trie()
										trie.add("ABC"      )
										trie.add("B"      )
										trie.add("CD"      )
										self.assertEqual(trie.split("ABCD"      )       ,  ["ABC", "D"]      )



							def      snake_case							(	self      ):
										# Even if the offsets are wrong, we necessarily output correct string
										# parts.
										__lowerCAmelCase							       =     Trie()
										__lowerCAmelCase							       =     trie.cut_text("ABC"       ,  [0, 0, 2, 1, 2, 3]      )
										self.assertEqual(__a       ,  ["AB", "C"]      )








"""simple docstring"""


import warnings

from ...utils import logging
from .image_processing_yolos import YolosImageProcessor


A	:  Optional[Any]            =     logging.get_logger(__name__)
class     _UpperCamelCase    (  lowerCAmelCase__						):
							'''simple docstring'''







							def __init__(	self       ,  *__a       ,  **__a      ):
										warnings.warn(
										    "The class YolosFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
										    " use YolosImageProcessor instead."       ,  __a       ,  )
										super().__init__(*__a       ,  **__a      )




