






"""simple docstring"""






import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast




class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE (						self     :  Optional[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		mock.Mock()
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		500
          SCREAMING_SNAKE_CASE_      :  Any    						=		{}
          SCREAMING_SNAKE_CASE_      :  str    						=		HTTPError
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		{}

          # Download this model to make sure it's in the cache.
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('''requests.Session.request'''						,	return_value=lowercase_) as mock_head:
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
              # This check we did call the fake head request
              mock_head.assert_called()
      @require_tokenizers
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		mock.Mock()
          SCREAMING_SNAKE_CASE_      :  int    						=		500
          SCREAMING_SNAKE_CASE_      :  Any    						=		{}
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		HTTPError
          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		{}

          # Download this model to make sure it's in the cache.
          SCREAMING_SNAKE_CASE_      :  Any    						=		GPTaTokenizerFast.from_pretrained('''gpt2''')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('''requests.Session.request'''						,	return_value=lowercase_) as mock_head:
              SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		GPTaTokenizerFast.from_pretrained('''gpt2''')
              # This check we did call the fake head request
              mock_head.assert_called()
      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]):




          '''simple docstring'''


          try:
              SCREAMING_SNAKE_CASE_      :  Dict    						=		tempfile.mktemp()
              with open(lowercase_						,	'''wb''') as f:
                  http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''						,	lowercase_)

              SCREAMING_SNAKE_CASE_      :  str    						=		AlbertTokenizer.from_pretrained(lowercase_)
          finally:
              os.remove(lowercase_)

          # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
          # the current folder and have the right name.
          if os.path.isfile('''tokenizer.json'''):
              # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
              return
          try:
              with open('''tokenizer.json'''						,	'''wb''') as f:
                  http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''						,	lowercase_)
              SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''')
              # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
              self.assertEqual(tokenizer.vocab_size						,	1000)
              # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

          finally:
              os.remove('''tokenizer.json''')



      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  str    						=		AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''')




@is_staging_test
class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      __UpperCamelCase	 =  ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]
      @classmethod
      def _SCREAMING_SNAKE_CASE (						cls     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		TOKEN
          HfFolder.save_token(lowercase_)
      @classmethod
      def _SCREAMING_SNAKE_CASE (						cls     :  str):




          '''simple docstring'''


          try:
              delete_repo(token=cls._token						,	repo_id='''test-tokenizer''')
          except HTTPError:
              pass

          try:
              delete_repo(token=cls._token						,	repo_id='''valid_org/test-tokenizer-org''')
          except HTTPError:
              pass

          try:
              delete_repo(token=cls._token						,	repo_id='''test-dynamic-tokenizer''')
          except HTTPError:
              pass
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          with tempfile.TemporaryDirectory() as tmp_dir:
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		os.path.join(lowercase_						,	'''vocab.txt''')
              with open(lowercase_						,	'''w'''						,	encoding='''utf-8''') as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
              SCREAMING_SNAKE_CASE_      :  int    						=		BertTokenizer(lowercase_)

          tokenizer.push_to_hub('''test-tokenizer'''						,	use_auth_token=self._token)
          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab						,	tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token						,	repo_id='''test-tokenizer''')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(lowercase_						,	repo_id='''test-tokenizer'''						,	push_to_hub=lowercase_						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Dict    						=		BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab						,	tokenizer.vocab)
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          with tempfile.TemporaryDirectory() as tmp_dir:
              SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		os.path.join(lowercase_						,	'''vocab.txt''')
              with open(lowercase_						,	'''w'''						,	encoding='''utf-8''') as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		BertTokenizer(lowercase_)

          tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''						,	use_auth_token=self._token)
          SCREAMING_SNAKE_CASE_      :  Tuple    						=		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
          self.assertDictEqual(new_tokenizer.vocab						,	tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token						,	repo_id='''valid_org/test-tokenizer-org''')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(
                  lowercase_						,	repo_id='''valid_org/test-tokenizer-org'''						,	push_to_hub=lowercase_						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org''')
          self.assertDictEqual(new_tokenizer.vocab						,	tokenizer.vocab)



      @require_tokenizers
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          CustomTokenizer.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
              SCREAMING_SNAKE_CASE_      :  str    						=		os.path.join(lowercase_						,	'''vocab.txt''')
              with open(lowercase_						,	'''w'''						,	encoding='''utf-8''') as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))
              SCREAMING_SNAKE_CASE_      :  List[Any]    						=		CustomTokenizer(lowercase_)

          # No fast custom tokenizer
          tokenizer.push_to_hub('''test-dynamic-tokenizer'''						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  Any    						=		AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,	trust_remote_code=lowercase_)
          # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__						,	'''CustomTokenizer''')

          # Fast and slow custom tokenizer
          CustomTokenizerFast.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
              SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		os.path.join(lowercase_						,	'''vocab.txt''')
              with open(lowercase_						,	'''w'''						,	encoding='''utf-8''') as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]))

              SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		BertTokenizerFast.from_pretrained(lowercase_)
              bert_tokenizer.save_pretrained(lowercase_)
              SCREAMING_SNAKE_CASE_      :  List[str]    						=		CustomTokenizerFast.from_pretrained(lowercase_)

          tokenizer.push_to_hub('''test-dynamic-tokenizer'''						,	use_auth_token=self._token)

          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'						,	trust_remote_code=lowercase_)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__						,	'''CustomTokenizerFast''')
          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		AutoTokenizer.from_pretrained(
              F'{USER}/test-dynamic-tokenizer'						,	use_fast=lowercase_						,	trust_remote_code=lowercase_)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__						,	'''CustomTokenizer''')




class 							lowerCAmelCase__				(			unittest.TestCase ):






      '''simple docstring'''

      def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  int    						=		Trie()
          trie.add('''Hello 友達''')
          self.assertEqual(trie.data						,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})
          trie.add('''Hello''')
          trie.data
          self.assertEqual(trie.data						,	{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}})
      def _SCREAMING_SNAKE_CASE (						self     :  str):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		Trie()
          self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')						,	['''[CLS] This is a extra_id_100'''])
          trie.add('''[CLS]''')
          trie.add('''extra_id_1''')
          trie.add('''extra_id_100''')
          self.assertEqual(trie.split('''[CLS] This is a extra_id_100''')						,	['''[CLS]''', ''' This is a ''', '''extra_id_100'''])
      def _SCREAMING_SNAKE_CASE (						self     :  List[str]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[str]    						=		Trie()
          trie.add('''A''')
          self.assertEqual(trie.split('''ABC''')						,	['''A''', '''BC'''])
          self.assertEqual(trie.split('''BCA''')						,	['''BC''', '''A'''])
      def _SCREAMING_SNAKE_CASE (						self     :  Tuple):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Any    						=		Trie()
          trie.add('''TOKEN]''')
          trie.add('''[SPECIAL_TOKEN]''')
          self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')						,	['''This is something ''', '''[SPECIAL_TOKEN]'''])
      def _SCREAMING_SNAKE_CASE (						self     :  Any):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		Trie()
          trie.add('''A''')
          trie.add('''P''')
          trie.add('''[SPECIAL_TOKEN]''')
          self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]''')						,	['''This is something ''', '''[SPECIAL_TOKEN]'''])
      def _SCREAMING_SNAKE_CASE (						self     :  Optional[int]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  List[Any]    						=		Trie()
          trie.add('''AB''')
          trie.add('''B''')
          trie.add('''C''')
          self.assertEqual(trie.split('''ABC''')						,	['''AB''', '''C'''])
      def _SCREAMING_SNAKE_CASE (						self     :  Dict):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Dict    						=		Trie()
          trie.add('''ABC''')
          trie.add('''B''')
          trie.add('''CD''')
          self.assertEqual(trie.split('''ABCD''')						,	['''ABC''', '''D'''])



      def _SCREAMING_SNAKE_CASE (						self     :  List[Any]):




          '''simple docstring'''


          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		Trie()
          SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		trie.cut_text('''ABC'''						,	[0, 0, 2, 1, 2, 3])
          self.assertEqual(lowercase_						,	['''AB''', '''C'''])










"""simple docstring"""






from . import (
    albert,
    align,
    altclip,
    audio_spectrogram_transformer,
    auto,
    autoformer,
    bark,
    bart,
    barthez,
    bartpho,
    beit,
    bert,
    bert_generation,
    bert_japanese,
    bertweet,
    big_bird,
    bigbird_pegasus,
    biogpt,
    bit,
    blenderbot,
    blenderbot_small,
    blip,
    blip_a,
    bloom,
    bridgetower,
    byta,
    camembert,
    canine,
    chinese_clip,
    clap,
    clip,
    clipseg,
    codegen,
    conditional_detr,
    convbert,
    convnext,
    convnextva,
    cpm,
    cpmant,
    ctrl,
    cvt,
    dataavec,
    deberta,
    deberta_va,
    decision_transformer,
    deformable_detr,
    deit,
    deprecated,
    deta,
    detr,
    dialogpt,
    dinat,
    distilbert,
    dit,
    donut,
    dpr,
    dpt,
    efficientformer,
    efficientnet,
    electra,
    encodec,
    encoder_decoder,
    ernie,
    ernie_m,
    esm,
    falcon,
    flaubert,
    flava,
    fnet,
    focalnet,
    fsmt,
    funnel,
    git,
    glpn,
    gpta,
    gpt_bigcode,
    gpt_neo,
    gpt_neox,
    gpt_neox_japanese,
    gpt_swa,
    gptj,
    gptsan_japanese,
    graphormer,
    groupvit,
    herbert,
    hubert,
    ibert,
    imagegpt,
    informer,
    instructblip,
    jukebox,
    layoutlm,
    layoutlmva,
    layoutlmva,
    layoutxlm,
    led,
    levit,
    lilt,
    llama,
    longformer,
    longta,
    luke,
    lxmert,
    mam_aaa,
    marian,
    markuplm,
    maskaformer,
    maskformer,
    mbart,
    mbartaa,
    mega,
    megatron_bert,
    megatron_gpta,
    mgp_str,
    mluke,
    mobilebert,
    mobilenet_va,
    mobilenet_va,
    mobilevit,
    mobilevitva,
    mpnet,
    mra,
    mta,
    musicgen,
    mvp,
    nat,
    nezha,
    nllb,
    nllb_moe,
    nystromformer,
    oneformer,
    open_llama,
    openai,
    opt,
    owlvit,
    pegasus,
    pegasus_x,
    perceiver,
    phobert,
    pixastruct,
    plbart,
    poolformer,
    prophetnet,
    qdqbert,
    rag,
    realm,
    reformer,
    regnet,
    rembert,
    resnet,
    roberta,
    roberta_prelayernorm,
    roc_bert,
    roformer,
    rwkv,
    sam,
    segformer,
    sew,
    sew_d,
    speech_encoder_decoder,
    speech_to_text,
    speech_to_text_a,
    speechta,
    splinter,
    squeezebert,
    swiftformer,
    swin,
    swinasr,
    swinva,
    switch_transformers,
    ta,
    table_transformer,
    tapas,
    time_series_transformer,
    timesformer,
    timm_backbone,
    transfo_xl,
    trocr,
    tvlt,
    umta,
    unispeech,
    unispeech_sat,
    upernet,
    videomae,
    vilt,
    vision_encoder_decoder,
    vision_text_dual_encoder,
    visual_bert,
    vit,
    vit_hybrid,
    vit_mae,
    vit_msn,
    vivit,
    wavaveca,
    wavaveca_conformer,
    wavaveca_phoneme,
    wavaveca_with_lm,
    wavlm,
    whisper,
    x_clip,
    xglm,
    xlm,
    xlm_prophetnet,
    xlm_roberta,
    xlm_roberta_xl,
    xlnet,
    xmod,
    yolos,
    yoso,
)


