


"""simple docstring"""







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast


class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							mock.Mock()
         lowerCAmelCase     :     Any								=							500
         lowerCAmelCase     :     Optional[Any]								=							{}
         lowerCAmelCase     :     int								=							HTTPError
         lowerCAmelCase     :     Optional[int]								=							{}

         # Download this model to make sure it's in the cache.
         lowerCAmelCase     :     List[str]								=							BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch("requests.Session.request"		,     return_value=snake_case__						) as mock_head:
           lowerCAmelCase     :     Any								=							BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
           # This check we did call the fake head request
           mock_head.assert_called()

       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							mock.Mock()
         lowerCAmelCase     :     int								=							500
         lowerCAmelCase     :     Dict								=							{}
         lowerCAmelCase     :     List[Any]								=							HTTPError
         lowerCAmelCase     :     Any								=							{}

         # Download this model to make sure it's in the cache.
         lowerCAmelCase     :     Optional[Any]								=							GPTaTokenizerFast.from_pretrained("gpt2"						)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch("requests.Session.request"		,     return_value=snake_case__						) as mock_head:
           lowerCAmelCase     :     Any								=							GPTaTokenizerFast.from_pretrained("gpt2"						)
           # This check we did call the fake head request
           mock_head.assert_called()

       def        lowercase__							(						self						):






         """simple docstring"""





         try:
           lowerCAmelCase     :     Optional[int]								=							tempfile.mktemp()
           with open(snake_case__		,     "wb"						) as f:
             http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"		,     snake_case__						)

           lowerCAmelCase     :     Union[str, Any]								=							AlbertTokenizer.from_pretrained(snake_case__						)
         finally:
           os.remove(snake_case__						)

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile("tokenizer.json"						):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
         try:
           with open("tokenizer.json"		,     "wb"						) as f:
             http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"		,     snake_case__						)
           lowerCAmelCase     :     int								=							AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"						)
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size		,     1_000						)
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
           os.remove("tokenizer.json"						)







       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"						)


@is_staging_test
class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       a  :				Any   =["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

       @classmethod
       def        lowercase__							(						cls						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							TOKEN
         HfFolder.save_token(snake_case__						)

       @classmethod
       def        lowercase__							(						cls						):






         """simple docstring"""





         try:
           delete_repo(token=cls._token		,     repo_id="test-tokenizer"						)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token		,     repo_id="valid_org/test-tokenizer-org"						)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token		,     repo_id="test-dynamic-tokenizer"						)
         except HTTPError:
           pass

       def        lowercase__							(						self						):






         """simple docstring"""





         with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     :     Optional[Any]								=							os.path.join(snake_case__		,     "vocab.txt"						)
           with open(snake_case__		,     "w"		,     encoding="utf-8"						) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
           lowerCAmelCase     :     Optional[Any]								=							BertTokenizer(snake_case__						)

         tokenizer.push_to_hub("test-tokenizer"		,     use_auth_token=self._token						)
         lowerCAmelCase     :     List[Any]								=							BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""						)
         self.assertDictEqual(new_tokenizer.vocab		,     tokenizer.vocab						)

         # Reset repo
         delete_repo(token=self._token		,     repo_id="test-tokenizer"						)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(snake_case__		,     repo_id="test-tokenizer"		,     push_to_hub=snake_case__		,     use_auth_token=self._token						)

         lowerCAmelCase     :     Any								=							BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""						)
         self.assertDictEqual(new_tokenizer.vocab		,     tokenizer.vocab						)

       def        lowercase__							(						self						):






         """simple docstring"""





         with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     :     str								=							os.path.join(snake_case__		,     "vocab.txt"						)
           with open(snake_case__		,     "w"		,     encoding="utf-8"						) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
           lowerCAmelCase     :     Tuple								=							BertTokenizer(snake_case__						)

         tokenizer.push_to_hub("valid_org/test-tokenizer-org"		,     use_auth_token=self._token						)
         lowerCAmelCase     :     Any								=							BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
         self.assertDictEqual(new_tokenizer.vocab		,     tokenizer.vocab						)

         # Reset repo
         delete_repo(token=self._token		,     repo_id="valid_org/test-tokenizer-org"						)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               snake_case__		,     repo_id="valid_org/test-tokenizer-org"		,     push_to_hub=snake_case__		,     use_auth_token=self._token						)

         lowerCAmelCase     :     Union[str, Any]								=							BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
         self.assertDictEqual(new_tokenizer.vocab		,     tokenizer.vocab						)







       @require_tokenizers
       def        lowercase__							(						self						):






         """simple docstring"""





         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     :     List[str]								=							os.path.join(snake_case__		,     "vocab.txt"						)
           with open(snake_case__		,     "w"		,     encoding="utf-8"						) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
           lowerCAmelCase     :     Tuple								=							CustomTokenizer(snake_case__						)

         # No fast custom tokenizer
         tokenizer.push_to_hub("test-dynamic-tokenizer"		,     use_auth_token=self._token						)

         lowerCAmelCase     :     Optional[Any]								=							AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""		,     trust_remote_code=snake_case__						)
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__		,     "CustomTokenizer"						)

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           lowerCAmelCase     :     List[Any]								=							os.path.join(snake_case__		,     "vocab.txt"						)
           with open(snake_case__		,     "w"		,     encoding="utf-8"						) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)

           lowerCAmelCase     :     str								=							BertTokenizerFast.from_pretrained(snake_case__						)
           bert_tokenizer.save_pretrained(snake_case__						)
           lowerCAmelCase     :     Optional[int]								=							CustomTokenizerFast.from_pretrained(snake_case__						)

         tokenizer.push_to_hub("test-dynamic-tokenizer"		,     use_auth_token=self._token						)

         lowerCAmelCase     :     Dict								=							AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""		,     trust_remote_code=snake_case__						)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__		,     "CustomTokenizerFast"						)
         lowerCAmelCase     :     Union[str, Any]								=							AutoTokenizer.from_pretrained(
             f"""{USER}/test-dynamic-tokenizer"""		,     use_fast=snake_case__		,     trust_remote_code=snake_case__						)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__		,     "CustomTokenizer"						)


class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[Any]								=							Trie()
         trie.add("Hello 友達"						)
         self.assertEqual(trie.data		,     {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}						)
         trie.add("Hello"						)
         trie.data
         self.assertEqual(trie.data		,     {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     int								=							Trie()
         self.assertEqual(trie.split("[CLS] This is a extra_id_100"						)		,     ["[CLS] This is a extra_id_100"]						)
         trie.add("[CLS]"						)
         trie.add("extra_id_1"						)
         trie.add("extra_id_100"						)
         self.assertEqual(trie.split("[CLS] This is a extra_id_100"						)		,     ["[CLS]", " This is a ", "extra_id_100"]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     str								=							Trie()
         trie.add("A"						)
         self.assertEqual(trie.split("ABC"						)		,     ["A", "BC"]						)
         self.assertEqual(trie.split("BCA"						)		,     ["BC", "A"]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     List[str]								=							Trie()
         trie.add("TOKEN]"						)
         trie.add("[SPECIAL_TOKEN]"						)
         self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						)		,     ["This is something ", "[SPECIAL_TOKEN]"]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							Trie()
         trie.add("A"						)
         trie.add("P"						)
         trie.add("[SPECIAL_TOKEN]"						)
         self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						)		,     ["This is something ", "[SPECIAL_TOKEN]"]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Dict								=							Trie()
         trie.add("AB"						)
         trie.add("B"						)
         trie.add("C"						)
         self.assertEqual(trie.split("ABC"						)		,     ["AB", "C"]						)

       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							Trie()
         trie.add("ABC"						)
         trie.add("B"						)
         trie.add("CD"						)
         self.assertEqual(trie.split("ABCD"						)		,     ["ABC", "D"]						)







       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Optional[int]								=							Trie()
         lowerCAmelCase     :     Optional[int]								=							trie.cut_text("ABC"		,     [0, 0, 2, 1, 2, 3]						)
         self.assertEqual(snake_case__		,     ["AB", "C"]						)









"""simple docstring"""







lowerCAmelCase__      				=   [
    '''Audio''',
    '''Array2D''',
    '''Array3D''',
    '''Array4D''',
    '''Array5D''',
    '''ClassLabel''',
    '''Features''',
    '''Sequence''',
    '''Value''',
    '''Image''',
    '''Translation''',
    '''TranslationVariableLanguages''',
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages





