





"""simple docstring"""




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast

class  _UpperCAmelCase  (   unittest.TestCase		):



      '''simple docstring'''

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Optional[int]										=  mock.Mock()
        A_  :       int										=  5_0_0
        A_  :       Dict										=  {}
        A_  :       Tuple										=  HTTPError
        A_  :       Optional[Any]										=  {}

        # Download this model to make sure it's in the cache.
        A_  :       Tuple										=  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'  )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('requests.Session.request'					,      return_value=snake_case_  ) as mock_head:
          A_  :       List[str]										=  BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'  )
          # This check we did call the fake head request
          mock_head.assert_called()

      @require_tokenizers
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       List[Any]										=  mock.Mock()
        A_  :       List[str]										=  5_0_0
        A_  :       str										=  {}
        A_  :       Optional[Any]										=  HTTPError
        A_  :       Any										=  {}

        # Download this model to make sure it's in the cache.
        A_  :       List[str]										=  GPTaTokenizerFast.from_pretrained('gpt2'  )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('requests.Session.request'					,      return_value=snake_case_  ) as mock_head:
          A_  :       Any										=  GPTaTokenizerFast.from_pretrained('gpt2'  )
          # This check we did call the fake head request
          mock_head.assert_called()

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        try:
          A_  :       Any										=  tempfile.mktemp()
          with open(snake_case_					,      'wb'  ) as f:
            http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'					,      snake_case_  )

          A_  :       Optional[Any]										=  AlbertTokenizer.from_pretrained(snake_case_  )
        finally:
          os.remove(snake_case_  )

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('tokenizer.json'  ):
          # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
          return
        try:
          with open('tokenizer.json'					,      'wb'  ) as f:
            http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'					,      snake_case_  )
          A_  :       Optional[int]										=  AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'  )
          # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
          self.assertEqual(tokenizer.vocab_size					,      1_0_0_0  )
          # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
          os.remove('tokenizer.json'  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Tuple										=  AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  )


@is_staging_test
class  _UpperCAmelCase  (   unittest.TestCase		):



      '''simple docstring'''
      lowercase_      : str	    =   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]

      @classmethod
      def 					lowerCamelCase_			(     cls  ):

        """simple docstring"""

        A_  :       str										=  TOKEN
        HfFolder.save_token(snake_case_  )

      @classmethod
      def 					lowerCamelCase_			(     cls  ):

        """simple docstring"""

        try:
          delete_repo(token=cls._token					,      repo_id='test-tokenizer'  )
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token					,      repo_id='valid_org/test-tokenizer-org'  )
        except HTTPError:
          pass

        try:
          delete_repo(token=cls._token					,      repo_id='test-dynamic-tokenizer'  )
        except HTTPError:
          pass

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        with tempfile.TemporaryDirectory() as tmp_dir:
          A_  :       Tuple										=  os.path.join(snake_case_					,      'vocab.txt'  )
          with open(snake_case_					,      'w'					,      encoding='utf-8'  ) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
          A_  :       str										=  BertTokenizer(snake_case_  )

        tokenizer.push_to_hub('test-tokenizer'					,      use_auth_token=self._token  )
        A_  :       Optional[Any]										=  BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
        self.assertDictEqual(new_tokenizer.vocab					,      tokenizer.vocab  )

        # Reset repo
        delete_repo(token=self._token					,      repo_id='test-tokenizer'  )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(snake_case_					,      repo_id='test-tokenizer'					,      push_to_hub=snake_case_					,      use_auth_token=self._token  )

        A_  :       List[Any]										=  BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""  )
        self.assertDictEqual(new_tokenizer.vocab					,      tokenizer.vocab  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        with tempfile.TemporaryDirectory() as tmp_dir:
          A_  :       Tuple										=  os.path.join(snake_case_					,      'vocab.txt'  )
          with open(snake_case_					,      'w'					,      encoding='utf-8'  ) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
          A_  :       List[str]										=  BertTokenizer(snake_case_  )

        tokenizer.push_to_hub('valid_org/test-tokenizer-org'					,      use_auth_token=self._token  )
        A_  :       str										=  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'  )
        self.assertDictEqual(new_tokenizer.vocab					,      tokenizer.vocab  )

        # Reset repo
        delete_repo(token=self._token					,      repo_id='valid_org/test-tokenizer-org'  )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(
              snake_case_					,      repo_id='valid_org/test-tokenizer-org'					,      push_to_hub=snake_case_					,      use_auth_token=self._token  )

        A_  :       Optional[Any]										=  BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'  )
        self.assertDictEqual(new_tokenizer.vocab					,      tokenizer.vocab  )

      @require_tokenizers
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
          A_  :       Optional[int]										=  os.path.join(snake_case_					,      'vocab.txt'  )
          with open(snake_case_					,      'w'					,      encoding='utf-8'  ) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )
          A_  :       Dict										=  CustomTokenizer(snake_case_  )

        # No fast custom tokenizer
        tokenizer.push_to_hub('test-dynamic-tokenizer'					,      use_auth_token=self._token  )

        A_  :       Union[str, Any]										=  AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""					,      trust_remote_code=snake_case_  )
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__					,      'CustomTokenizer'  )

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
          A_  :       Any										=  os.path.join(snake_case_					,      'vocab.txt'  )
          with open(snake_case_					,      'w'					,      encoding='utf-8'  ) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]  )  )

          A_  :       Tuple										=  BertTokenizerFast.from_pretrained(snake_case_  )
          bert_tokenizer.save_pretrained(snake_case_  )
          A_  :       str										=  CustomTokenizerFast.from_pretrained(snake_case_  )

        tokenizer.push_to_hub('test-dynamic-tokenizer'					,      use_auth_token=self._token  )

        A_  :       Any										=  AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""					,      trust_remote_code=snake_case_  )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__					,      'CustomTokenizerFast'  )
        A_  :       List[str]										=  AutoTokenizer.from_pretrained(
            F"""{USER}/test-dynamic-tokenizer"""					,      use_fast=snake_case_					,      trust_remote_code=snake_case_  )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__					,      'CustomTokenizer'  )


class  _UpperCAmelCase  (   unittest.TestCase		):



      '''simple docstring'''

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       List[Any]										=  Trie()
        trie.add('Hello 友達'  )
        self.assertEqual(trie.data					,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}  )
        trie.add('Hello'  )
        trie.data
        self.assertEqual(trie.data					,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Optional[int]										=  Trie()
        self.assertEqual(trie.split('[CLS] This is a extra_id_100'  )					,      ['[CLS] This is a extra_id_100']  )
        trie.add('[CLS]'  )
        trie.add('extra_id_1'  )
        trie.add('extra_id_100'  )
        self.assertEqual(trie.split('[CLS] This is a extra_id_100'  )					,      ['[CLS]', ' This is a ', 'extra_id_100']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       int										=  Trie()
        trie.add('A'  )
        self.assertEqual(trie.split('ABC'  )					,      ['A', 'BC']  )
        self.assertEqual(trie.split('BCA'  )					,      ['BC', 'A']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Dict										=  Trie()
        trie.add('TOKEN]'  )
        trie.add('[SPECIAL_TOKEN]'  )
        self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'  )					,      ['This is something ', '[SPECIAL_TOKEN]']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       Optional[Any]										=  Trie()
        trie.add('A'  )
        trie.add('P'  )
        trie.add('[SPECIAL_TOKEN]'  )
        self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'  )					,      ['This is something ', '[SPECIAL_TOKEN]']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       List[Any]										=  Trie()
        trie.add('AB'  )
        trie.add('B'  )
        trie.add('C'  )
        self.assertEqual(trie.split('ABC'  )					,      ['AB', 'C']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       List[Any]										=  Trie()
        trie.add('ABC'  )
        trie.add('B'  )
        trie.add('CD'  )
        self.assertEqual(trie.split('ABCD'  )					,      ['ABC', 'D']  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        A_  :       int										=  Trie()
        A_  :       List[Any]										=  trie.cut_text('ABC'					,      [0, 0, 2, 1, 2, 3]  )
        self.assertEqual(snake_case_					,      ['AB', 'C']  )






"""simple docstring"""




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_     :						str						=					logging.get_logger(__name__)

lowerCamelCase_     :						Any						=					{
    'facebook/xmod-base': 'https://huggingface.co/facebook/xmod-base/resolve/main/config.json',
    'facebook/xmod-large-prenorm': 'https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json',
    'facebook/xmod-base-13-125k': 'https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json',
    'facebook/xmod-base-30-125k': 'https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json',
    'facebook/xmod-base-30-195k': 'https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json',
    'facebook/xmod-base-60-125k': 'https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json',
    'facebook/xmod-base-60-265k': 'https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json',
    'facebook/xmod-base-75-125k': 'https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json',
    'facebook/xmod-base-75-269k': 'https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json',
}

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : Tuple	    =   """xmod"""

      def __init__(     self					,      snake_case_=3_0_5_2_2					,      snake_case_=7_6_8					,      snake_case_=1_2					,      snake_case_=1_2					,      snake_case_=3_0_7_2					,      snake_case_="gelu"					,      snake_case_=0.1					,      snake_case_=0.1					,      snake_case_=5_1_2					,      snake_case_=2					,      snake_case_=0.02					,      snake_case_=1E-12					,      snake_case_=1					,      snake_case_=0					,      snake_case_=2					,      snake_case_="absolute"					,      snake_case_=True					,      snake_case_=None					,      snake_case_=False					,      snake_case_=2					,      snake_case_=False					,      snake_case_=True					,      snake_case_=True					,      snake_case_=("en_XX",)					,      snake_case_=None					,      **snake_case_					,      ):

        """simple docstring"""

        super().__init__(pad_token_id=snake_case_					,      bos_token_id=snake_case_					,      eos_token_id=snake_case_					,      **snake_case_  )

        A_  :       Union[str, Any]										=  vocab_size
        A_  :       Any										=  hidden_size
        A_  :       List[str]										=  num_hidden_layers
        A_  :       Tuple										=  num_attention_heads
        A_  :       int										=  hidden_act
        A_  :       Any										=  intermediate_size
        A_  :       Any										=  hidden_dropout_prob
        A_  :       Dict										=  attention_probs_dropout_prob
        A_  :       Union[str, Any]										=  max_position_embeddings
        A_  :       List[Any]										=  type_vocab_size
        A_  :       List[str]										=  initializer_range
        A_  :       Any										=  layer_norm_eps
        A_  :       Optional[Any]										=  position_embedding_type
        A_  :       int										=  use_cache
        A_  :       Dict										=  classifier_dropout
        A_  :       int										=  pre_norm
        A_  :       Optional[Any]										=  adapter_reduction_factor
        A_  :       List[Any]										=  adapter_layer_norm
        A_  :       int										=  adapter_reuse_layer_norm
        A_  :       Dict										=  ln_before_adapter
        A_  :       List[str]										=  list(snake_case_  )
        A_  :       Union[str, Any]										=  default_language


class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''

      @property
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        if self.task == "multiple-choice":
          A_  :       Dict										=  {0: 'batch', 1: 'choice', 2: 'sequence'}
        else:
          A_  :       int										=  {0: 'batch', 1: 'sequence'}
        return OrderedDict(
            [
                ('input_ids', dynamic_axis),
                ('attention_mask', dynamic_axis),
            ]  )