


'''simple docstring'''







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast







class A__       (						unittest.TestCase      ):






      def __UpperCAmelCase       ( self			:List[str]	)       ->					List[Any]:




             '''simple docstring'''
             _a    : Optional[int]     					=0






      @slow
      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					List[str]:




             '''simple docstring'''
             for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                    _a    : int     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
                    self.assertIsNotNone(SCREAMING_SNAKE_CASE	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(BertTokenizer, BertTokenizerFast)	)
                    self.assertGreater(len(SCREAMING_SNAKE_CASE	)     ,						0	)

             for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                    _a    : List[str]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
                    self.assertIsNotNone(SCREAMING_SNAKE_CASE	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(GPTaTokenizer, GPTaTokenizerFast)	)
                    self.assertGreater(len(SCREAMING_SNAKE_CASE	)     ,						0	)






      def __UpperCAmelCase       ( self			:int	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Union[str, Any]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(BertTokenizer, BertTokenizerFast)	)
             self.assertEqual(tokenizer.vocab_size     ,						1_2	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Dict     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(RobertaTokenizer, RobertaTokenizerFast)	)
             self.assertEqual(tokenizer.vocab_size     ,						2_0	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					List[str]:




             '''simple docstring'''
             _a    : Any     					=AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             # Check that tokenizer_type ≠ model_type
             _a    : Optional[Any]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						config=SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(BertTokenizer, BertTokenizerFast)	)
             self.assertEqual(tokenizer.vocab_size     ,						1_2	)






      def __UpperCAmelCase       ( self			:Tuple	)       ->					Dict:




             '''simple docstring'''
             with tempfile.TemporaryDirectory() as tmp_dir:
                    shutil.copy("""./tests/fixtures/vocab.txt"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""vocab.txt"""	)	)

                    _a    : List[Any]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						tokenizer_type="""bert"""     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    shutil.copy("""./tests/fixtures/vocab.json"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""vocab.json"""	)	)
                    shutil.copy("""./tests/fixtures/merges.txt"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""merges.txt"""	)	)

                    _a    : int     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						tokenizer_type="""gpt2"""     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:List[Any]	)       ->					int:




             '''simple docstring'''
             with tempfile.TemporaryDirectory() as tmp_dir:
                    shutil.copy("""./tests/fixtures/vocab.txt"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""vocab.txt"""	)	)

                    _a    : int     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						tokenizer_type="""bert"""	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             with tempfile.TemporaryDirectory() as tmp_dir:
                    shutil.copy("""./tests/fixtures/vocab.json"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""vocab.json"""	)	)
                    shutil.copy("""./tests/fixtures/merges.txt"""     ,						os.path.join(SCREAMING_SNAKE_CASE     ,						"""merges.txt"""	)	)

                    _a    : Optional[int]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						tokenizer_type="""gpt2"""	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Any:




             '''simple docstring'''
             with pytest.raises(SCREAMING_SNAKE_CASE	):
                    AutoTokenizer.from_pretrained("""./"""     ,						tokenizer_type="""xxx"""	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:Any	)       ->					str:




             '''simple docstring'''
             for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                    _a    : Optional[int]     					=tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""	)
                    self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(BertTokenizer, BertTokenizerFast)	)

                    if isinstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	):
                           self.assertEqual(tokenizer.basic_tokenizer.do_lower_case     ,						SCREAMING_SNAKE_CASE	)
                    else:
                           self.assertEqual(tokenizer.do_lower_case     ,						SCREAMING_SNAKE_CASE	)

                    self.assertEqual(tokenizer.model_max_length     ,						5_1_2	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					int:




             '''simple docstring'''
             for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                    with self.assertRaisesRegex(
                        SCREAMING_SNAKE_CASE     ,						"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""     ,						):
                           _a    : Any     					=tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""	)






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					List[Any]:




             '''simple docstring'''
             # tests: https://github.com/huggingface/transformers/pull/13251
             # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
             # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
             _a    : Dict     					=TOKENIZER_MAPPING.values()
             _a    : Tuple     					=[]

             for slow_tok, fast_tok in tokenizers:
                    if slow_tok is not None:
                           tokenizer_names.append(slow_tok.__name__	)

                    if fast_tok is not None:
                           tokenizer_names.append(fast_tok.__name__	)

             for tokenizer_name in tokenizer_names:
                    # must find the right class
                    tokenizer_class_from_name(SCREAMING_SNAKE_CASE	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:int	)       ->					List[str]:




             '''simple docstring'''
             self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""     ,						use_fast=SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""	)     ,						SCREAMING_SNAKE_CASE	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:int	)       ->					int:




             '''simple docstring'''
             _a    : List[str]     					=AutoTokenizer.from_pretrained("""distilbert-base-uncased"""     ,						do_lower_case=SCREAMING_SNAKE_CASE	)
             _a    : List[Any]     					="""Hello, world. How are you?"""
             _a    : Optional[int]     					=tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
             self.assertEqual("""[UNK]"""     ,						tokens[0]	)

             _a    : Optional[int]     					=AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""     ,						do_lower_case=SCREAMING_SNAKE_CASE	)
             _a    : int     					=tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
             self.assertEqual("""[UNK]"""     ,						tokens[0]	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:Tuple	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : Optional[int]     					=AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""	)
             self.assertEqual(type(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)
             self.assertEqual(tokenizer.model_max_length     ,						5_1_2	)
             self.assertEqual(tokenizer.vocab_size     ,						3_0_0_0_0	)
             self.assertEqual(tokenizer.unk_token     ,						"""[UNK]"""	)
             self.assertEqual(tokenizer.padding_side     ,						"""right"""	)
             self.assertEqual(tokenizer.truncation_side     ,						"""right"""	)






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Any:




             '''simple docstring'''
             _a    : int     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						(BertTokenizer, BertTokenizerFast)	)
             with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
                    _a    : Optional[Any]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)

             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						tokenizer.__class__	)
             self.assertEqual(tokenizera.vocab_size     ,						1_2	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					List[Any]:




             '''simple docstring'''
             _a    : str     					=AutoTokenizer.from_pretrained("""ctrl"""	)
             # There is no fast CTRL so this always gives us a slow tokenizer.
             self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Union[str, Any]:




             '''simple docstring'''
             # Check we can load the tokenizer config of an online model.
             _a    : Optional[int]     					=get_tokenizer_config("""bert-base-cased"""	)
             _a    : Any     					=config.pop("""_commit_hash"""     ,						SCREAMING_SNAKE_CASE	)
             # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						{"""do_lower_case""": False}	)

             # This model does not have a tokenizer_config so we get back an empty dict.
             _a    : List[str]     					=get_tokenizer_config(SCREAMING_SNAKE_CASE	)
             self.assertDictEqual(SCREAMING_SNAKE_CASE     ,						{}	)

             # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
             _a    : str     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
             with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
                    _a    : Any     					=get_tokenizer_config(SCREAMING_SNAKE_CASE	)

             # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
             self.assertEqual(config["""tokenizer_class"""]     ,						"""BertTokenizer"""	)






      def __UpperCAmelCase       ( self			:Optional[Any]	)       ->					Any:




             '''simple docstring'''
             try:
                    AutoConfig.register("""custom"""     ,						SCREAMING_SNAKE_CASE	)

                    AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						slow_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(SCREAMING_SNAKE_CASE	):
                           AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						slow_tokenizer_class=SCREAMING_SNAKE_CASE	)

                    _a    : List[str]     					=CustomTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)

                           _a    : Any     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
                           self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]






      @require_tokenizers
      def __UpperCAmelCase       ( self			:List[str]	)       ->					Optional[Any]:




             '''simple docstring'''
             try:
                    AutoConfig.register("""custom"""     ,						SCREAMING_SNAKE_CASE	)

                    # Can register in two steps
                    AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						slow_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,						(CustomTokenizer, None)	)
                    AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						fast_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,						(CustomTokenizer, CustomTokenizerFast)	)

                    del TOKENIZER_MAPPING._extra_content[CustomConfig]
                    # Can register in one step
                    AutoTokenizer.register(
                        SCREAMING_SNAKE_CASE     ,						slow_tokenizer_class=SCREAMING_SNAKE_CASE     ,						fast_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(TOKENIZER_MAPPING[CustomConfig]     ,						(CustomTokenizer, CustomTokenizerFast)	)

                    # Trying to register something existing in the Transformers library will raise an error
                    with self.assertRaises(SCREAMING_SNAKE_CASE	):
                           AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						fast_tokenizer_class=SCREAMING_SNAKE_CASE	)

                    # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                    # and that model does not have a tokenizer.json
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           _a    : int     					=BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE	)
                           bert_tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
                           _a    : int     					=CustomTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE	)

                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)

                           _a    : Tuple     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE	)
                           self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

                           _a    : Any     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                           self.assertIsInstance(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]






      def __UpperCAmelCase       ( self			:int	)       ->					Tuple:




             '''simple docstring'''
             # If remote code is not set, we will time out when asking whether to load the model.
             with self.assertRaises(SCREAMING_SNAKE_CASE	):
                    _a    : Dict     					=AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	)
             # If remote code is disabled, we can't load this config.
             with self.assertRaises(SCREAMING_SNAKE_CASE	):
                    _a    : Optional[Any]     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)

             _a    : Any     					=AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)
             self.assertTrue(tokenizer.special_attribute_present	)
             # Test tokenizer can be reloaded.
             with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
                    _a    : Union[str, Any]     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)
             self.assertTrue(reloaded_tokenizer.special_attribute_present	)

             if is_tokenizers_available():
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)
                    self.assertEqual(reloaded_tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)

                    # Test we can also load the slow version
                    _a    : List[Any]     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertTrue(tokenizer.special_attribute_present	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    # Test tokenizer can be reloaded.
                    with tempfile.TemporaryDirectory() as tmp_dir:
                           tokenizer.save_pretrained(SCREAMING_SNAKE_CASE	)
                           _a    : Tuple     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						trust_remote_code=SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(reloaded_tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    self.assertTrue(reloaded_tokenizer.special_attribute_present	)
             else:
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    self.assertEqual(reloaded_tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)






      @require_tokenizers
      def __UpperCAmelCase       ( self			:List[str]	)       ->					Tuple:




             '''simple docstring'''







             class A__       (						UpperCAmelCase__      ):
                   __UpperCamelCase       :  str	       =							False







             class A__       (						UpperCAmelCase__      ):
                   __UpperCamelCase       :  List[str]	       =							NewTokenizer
                   __UpperCamelCase       :  Tuple	       =							False

             try:
                    AutoConfig.register("""custom"""     ,						SCREAMING_SNAKE_CASE	)
                    AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						slow_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    AutoTokenizer.register(SCREAMING_SNAKE_CASE     ,						fast_tokenizer_class=SCREAMING_SNAKE_CASE	)
                    # If remote code is not set, the default is to use local
                    _a    : str     					=AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)
                    self.assertFalse(tokenizer.special_attribute_present	)
                    _a    : Optional[int]     					=AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    self.assertFalse(tokenizer.special_attribute_present	)

                    # If remote code is disabled, we load the local one.
                    _a    : Tuple     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)
                    self.assertFalse(tokenizer.special_attribute_present	)
                    _a    : List[str]     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    self.assertFalse(tokenizer.special_attribute_present	)

                    # If remote is enabled, we load from the Hub
                    _a    : Optional[int]     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)
                    self.assertTrue(tokenizer.special_attribute_present	)
                    _a    : Dict     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
                    self.assertTrue(tokenizer.special_attribute_present	)

             finally:
                    if "custom" in CONFIG_MAPPING._extra_content:
                           del CONFIG_MAPPING._extra_content["custom"]
                    if CustomConfig in TOKENIZER_MAPPING._extra_content:
                           del TOKENIZER_MAPPING._extra_content[CustomConfig]






      def __UpperCAmelCase       ( self			:List[str]	)       ->					Dict:




             '''simple docstring'''
             _a    : List[str]     					=AutoTokenizer.from_pretrained(
                 """hf-internal-testing/test_dynamic_tokenizer_legacy"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE	)
             self.assertTrue(tokenizer.special_attribute_present	)
             if is_tokenizers_available():
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizerFast"""	)

                    # Test we can also load the slow version
                    _a    : str     					=AutoTokenizer.from_pretrained(
                        """hf-internal-testing/test_dynamic_tokenizer_legacy"""     ,						trust_remote_code=SCREAMING_SNAKE_CASE     ,						use_fast=SCREAMING_SNAKE_CASE	)
                    self.assertTrue(tokenizer.special_attribute_present	)
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)
             else:
                    self.assertEqual(tokenizer.__class__.__name__     ,						"""NewTokenizer"""	)






      def __UpperCAmelCase       ( self			:int	)       ->					int:




             '''simple docstring'''
             with self.assertRaisesRegex(
                 SCREAMING_SNAKE_CASE     ,						"""bert-base is not a local folder and is not a valid model identifier"""	):
                    _a    : Dict     					=AutoTokenizer.from_pretrained("""bert-base"""	)






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					Dict:




             '''simple docstring'''
             with self.assertRaisesRegex(
                 SCREAMING_SNAKE_CASE     ,						r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""	):
                    _a    : int     					=AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE     ,						revision="""aaaaaa"""	)






      def __UpperCAmelCase       ( self			:Union[str, Any]	)       ->					Tuple:




             '''simple docstring'''
             # Make sure we have cached the tokenizer.
             _a    : int     					=AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
             with RequestCounter() as counter:
                    _a    : Optional[Any]     					=AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
                    self.assertEqual(counter.get_request_count     ,						0	)
                    self.assertEqual(counter.head_request_count     ,						1	)
                    self.assertEqual(counter.other_request_count     ,						0	)






'''simple docstring'''







from __future__ import annotations

import requests


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	str  )  ->			dict:
       _a    : Any     					=F"https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty"
       return requests.get(_UpperCAmelCase  ).json()


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	int = 10  )  ->			list[dict]:
       _a    : Union[str, Any]     					="""https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"""
       _a    : int     					=requests.get(_UpperCAmelCase  ).json()[:max_stories]
       return [get_hackernews_story(_UpperCAmelCase  ) for story_id in story_ids]


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	int = 10  )  ->			str:
       _a    : Union[str, Any]     					=hackernews_top_stories(_UpperCAmelCase  )
       return "\n".join("""* [{title}]({url})""".format(**_UpperCAmelCase  ) for story in stories  )


if __name__ == "__main__":
   print(hackernews_top_stories_as_markdown())


