codingutf8 2018 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license auto tokenizer class import importlib import json import os import warnings from collections import ordereddict from typing import typechecking dict optional tuple union from configurationutils import pretrainedconfig from dynamicmoduleutils import getclassfromdynamicmodule resolvetrustremotecode from tokenizationutils import pretrainedtokenizer from tokenizationutilsbase import tokenizerconfigfile from utils import cachedfile extractcommithash issentencepieceavailable istokenizersavailable logging from encoderdecoder import encoderdecoderconfig from autofactory import lazyautomapping from configurationauto import configmappingnames autoconfig configclasstomodeltype modeltypetomodulename replacelistoptionindocstrings if istokenizersavailable from tokenizationutilsfast import pretrainedtokenizerfast else pretrainedtokenizerfast none logger logging getloggername if typechecking this significantly improves completion suggestion performance when the transformers package is used with microsoft s pylance language server tokenizermappingnames ordereddictstr tupleoptionalstr optionalstr ordereddict else tokenizermappingnames ordereddict albert alberttokenizer if issentencepieceavailable else none alberttokenizerfast if istokenizersavailable else none align berttokenizer berttokenizerfast if istokenizersavailable else none bark berttokenizer berttokenizerfast if istokenizersavailable else none bart barttokenizer barttokenizerfast barthez bartheztokenizer if issentencepieceavailable else none bartheztokenizerfast if istokenizersavailable else none bartpho bartphotokenizer none bert berttokenizer berttokenizerfast if istokenizersavailable else none bertgeneration bertgenerationtokenizer if issentencepieceavailable else none none bertjapanese bertjapanesetokenizer none bertweet bertweettokenizer none bigbird bigbirdtokenizer if issentencepieceavailable else none bigbirdtokenizerfast if istokenizersavailable else none bigbirdpegasus pegasustokenizer pegasustokenizerfast if istokenizersavailable else none biogpt biogpttokenizer none blenderbot blenderbottokenizer blenderbottokenizerfast blenderbotsmall blenderbotsmalltokenizer none blip berttokenizer berttokenizerfast if istokenizersavailable else none blip2 gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none bloom none bloomtokenizerfast if istokenizersavailable else none bridgetower robertatokenizer robertatokenizerfast if istokenizersavailable else none bros berttokenizer berttokenizerfast if istokenizersavailable else none byt5 byt5tokenizer none camembert camemberttokenizer if issentencepieceavailable else none camemberttokenizerfast if istokenizersavailable else none canine caninetokenizer none chineseclip berttokenizer berttokenizerfast if istokenizersavailable else none clap robertatokenizer robertatokenizerfast if istokenizersavailable else none clip cliptokenizer cliptokenizerfast if istokenizersavailable else none clipseg cliptokenizer cliptokenizerfast if istokenizersavailable else none clvp clvptokenizer none codellama codellamatokenizer if issentencepieceavailable else none codellamatokenizerfast if istokenizersavailable else none codegen codegentokenizer codegentokenizerfast if istokenizersavailable else none convbert convberttokenizer convberttokenizerfast if istokenizersavailable else none cpm cpmtokenizer if issentencepieceavailable else none cpmtokenizerfast if istokenizersavailable else none cpmant cpmanttokenizer none ctrl ctrltokenizer none data2vecaudio wav2vec2ctctokenizer none data2vectext robertatokenizer robertatokenizerfast if istokenizersavailable else none deberta debertatokenizer debertatokenizerfast if istokenizersavailable else none debertav2 debertav2tokenizer if issentencepieceavailable else none debertav2tokenizerfast if istokenizersavailable else none distilbert distilberttokenizer distilberttokenizerfast if istokenizersavailable else none dpr dprquestionencodertokenizer dprquestionencodertokenizerfast if istokenizersavailable else none electra electratokenizer electratokenizerfast if istokenizersavailable else none ernie berttokenizer berttokenizerfast if istokenizersavailable else none erniem erniemtokenizer if issentencepieceavailable else none none esm esmtokenizer none falcon none pretrainedtokenizerfast if istokenizersavailable else none flaubert flauberttokenizer none fnet fnettokenizer fnettokenizerfast if istokenizersavailable else none fsmt fsmttokenizer none funnel funneltokenizer funneltokenizerfast if istokenizersavailable else none git berttokenizer berttokenizerfast if istokenizersavailable else none gptsw3 gptsw3tokenizer if issentencepieceavailable else none none gpt2 gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none gptbigcode gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none gptneo gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none gptneox none gptneoxtokenizerfast if istokenizersavailable else none gptneoxjapanese gptneoxjapanesetokenizer none gptj gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none gptsanjapanese gptsanjapanesetokenizer none groupvit cliptokenizer cliptokenizerfast if istokenizersavailable else none herbert herberttokenizer herberttokenizerfast if istokenizersavailable else none hubert wav2vec2ctctokenizer none ibert robertatokenizer robertatokenizerfast if istokenizersavailable else none idefics none llamatokenizerfast if istokenizersavailable else none instructblip gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none jukebox jukeboxtokenizer none kosmos2 xlmrobertatokenizer if issentencepieceavailable else none xlmrobertatokenizerfast if istokenizersavailable else none layoutlm layoutlmtokenizer layoutlmtokenizerfast if istokenizersavailable else none layoutlmv2 layoutlmv2tokenizer layoutlmv2tokenizerfast if istokenizersavailable else none layoutlmv3 layoutlmv3tokenizer layoutlmv3tokenizerfast if istokenizersavailable else none layoutxlm layoutxlmtokenizer layoutxlmtokenizerfast if istokenizersavailable else none led ledtokenizer ledtokenizerfast if istokenizersavailable else none lilt layoutlmv3tokenizer layoutlmv3tokenizerfast if istokenizersavailable else none llama llamatokenizer if issentencepieceavailable else none llamatokenizerfast if istokenizersavailable else none longformer longformertokenizer longformertokenizerfast if istokenizersavailable else none longt5 t5tokenizer if issentencepieceavailable else none t5tokenizerfast if istokenizersavailable else none luke luketokenizer none lxmert lxmerttokenizer lxmerttokenizerfast if istokenizersavailable else none m2m100 m2m100tokenizer if issentencepieceavailable else none none marian mariantokenizer if issentencepieceavailable else none none mbart mbarttokenizer if issentencepieceavailable else none mbarttokenizerfast if istokenizersavailable else none mbart50 mbart50tokenizer if issentencepieceavailable else none mbart50tokenizerfast if istokenizersavailable else none mega robertatokenizer robertatokenizerfast if istokenizersavailable else none megatronbert berttokenizer berttokenizerfast if istokenizersavailable else none mgpstr mgpstrtokenizer none mistral llamatokenizer if issentencepieceavailable else none llamatokenizerfast if istokenizersavailable else none mluke mluketokenizer if issentencepieceavailable else none none mobilebert mobileberttokenizer mobileberttokenizerfast if istokenizersavailable else none mpnet mpnettokenizer mpnettokenizerfast if istokenizersavailable else none mpt none gptneoxtokenizerfast if istokenizersavailable else none mra robertatokenizer robertatokenizerfast if istokenizersavailable else none mt5 mt5tokenizer if issentencepieceavailable else none mt5tokenizerfast if istokenizersavailable else none musicgen t5tokenizer t5tokenizerfast if istokenizersavailable else none mvp mvptokenizer mvptokenizerfast if istokenizersavailable else none nezha berttokenizer berttokenizerfast if istokenizersavailable else none nllb nllbtokenizer if issentencepieceavailable else none nllbtokenizerfast if istokenizersavailable else none nllbmoe nllbtokenizer if issentencepieceavailable else none nllbtokenizerfast if istokenizersavailable else none nystromformer alberttokenizer if issentencepieceavailable else none alberttokenizerfast if istokenizersavailable else none oneformer cliptokenizer cliptokenizerfast if istokenizersavailable else none openaigpt openaigpttokenizer openaigpttokenizerfast if istokenizersavailable else none opt gpt2tokenizer gpt2tokenizerfast if istokenizersavailable else none owlv2 cliptokenizer cliptokenizerfast if istokenizersavailable else none owlvit cliptokenizer cliptokenizerfast if istokenizersavailable else none pegasus pegasustokenizer if issentencepieceavailable else none pegasustokenizerfast if istokenizersavailable else none pegasusx pegasustokenizer if issentencepieceavailable else none pegasustokenizerfast if istokenizersavailable else none perceiver perceivertokenizer none persimmon llamatokenizer if issentencepieceavailable else none llamatokenizerfast if istokenizersavailable else none phi codegentokenizer codegentokenizerfast if istokenizersavailable else none phobert phoberttokenizer none pix2struct t5tokenizer t5tokenizerfast if istokenizersavailable else none plbart plbarttokenizer if issentencepieceavailable else none none prophetnet prophetnettokenizer none qdqbert berttokenizer berttokenizerfast if istokenizersavailable else none rag ragtokenizer none realm realmtokenizer realmtokenizerfast if istokenizersavailable else none reformer reformertokenizer if issentencepieceavailable else none reformertokenizerfast if istokenizersavailable else none rembert remberttokenizer if issentencepieceavailable else none remberttokenizerfast if istokenizersavailable else none retribert retriberttokenizer retriberttokenizerfast if istokenizersavailable else none roberta robertatokenizer robertatokenizerfast if istokenizersavailable else none robertaprelayernorm robertatokenizer robertatokenizerfast if istokenizersavailable else none rocbert rocberttokenizer none roformer roformertokenizer roformertokenizerfast if istokenizersavailable else none rwkv none gptneoxtokenizerfast if istokenizersavailable else none seamlessm4t seamlessm4ttokenizer if issentencepieceavailable else none seamlessm4ttokenizerfast if istokenizersavailable else none seamlessm4tv2 seamlessm4ttokenizer if issentencepieceavailable else none seamlessm4ttokenizerfast if istokenizersavailable else none speechtotext speech2texttokenizer if issentencepieceavailable else none none speechtotext2 speech2text2tokenizer none speecht5 speecht5tokenizer if issentencepieceavailable else none none splinter splintertokenizer splintertokenizerfast squeezebert squeezeberttokenizer squeezeberttokenizerfast if istokenizersavailable else none switchtransformers t5tokenizer if issentencepieceavailable else none t5tokenizerfast if istokenizersavailable else none t5 t5tokenizer if issentencepieceavailable else none t5tokenizerfast if istokenizersavailable else none tapas tapastokenizer none tapex tapextokenizer none transfoxl transfoxltokenizer none tvp berttokenizer berttokenizerfast if istokenizersavailable else none umt5 t5tokenizer if issentencepieceavailable else none t5tokenizerfast if istokenizersavailable else none vilt berttokenizer berttokenizerfast if istokenizersavailable else none visualbert berttokenizer berttokenizerfast if istokenizersavailable else none vits vitstokenizer none wav2vec2 wav2vec2ctctokenizer none wav2vec2conformer wav2vec2ctctokenizer none wav2vec2phoneme wav2vec2phonemectctokenizer none whisper whispertokenizer whispertokenizerfast if istokenizersavailable else none xclip cliptokenizer cliptokenizerfast if istokenizersavailable else none xglm xglmtokenizer if issentencepieceavailable else none xglmtokenizerfast if istokenizersavailable else none xlm xlmtokenizer none xlmprophetnet xlmprophetnettokenizer if issentencepieceavailable else none none xlmroberta xlmrobertatokenizer if issentencepieceavailable else none xlmrobertatokenizerfast if istokenizersavailable else none xlmrobertaxl xlmrobertatokenizer if issentencepieceavailable else none xlmrobertatokenizerfast if istokenizersavailable else none xlnet xlnettokenizer if issentencepieceavailable else none xlnettokenizerfast if istokenizersavailable else none xmod xlmrobertatokenizer if issentencepieceavailable else none xlmrobertatokenizerfast if istokenizersavailable else none yoso alberttokenizer if issentencepieceavailable else none alberttokenizerfast if istokenizersavailable else none tokenizermapping lazyautomappingconfigmappingnames tokenizermappingnames configtotype v k for k v in configmappingnames items def tokenizerclassfromnameclassname str if classname pretrainedtokenizerfast return pretrainedtokenizerfast for modulename tokenizers in tokenizermappingnames items if classname in tokenizers modulename modeltypetomodulenamemodulename module importlib importmodulef modulename transformers models try return getattrmodule classname except attributeerror continue for config tokenizers in tokenizermapping extracontent items for tokenizer in tokenizers if getattrtokenizer name none classname return tokenizer we did not fine the class but maybe it s because a dep is missing in that case the class will be in the main init and we return the proper dummy to get an appropriate error message mainmodule importlib importmoduletransformers if hasattrmainmodule classname return getattrmainmodule classname return none def gettokenizerconfig pretrainedmodelnameorpath unionstr os pathlike cachedir optionalunionstr os pathlike none forcedownload bool false resumedownload bool false proxies optionaldictstr str none token optionalunionbool str none revision optionalstr none localfilesonly bool false subfolder str kwargs download configuration from huggingface co and cache this model does not have a tokenizer config so the result will be an empty dict save a pretrained tokenizer locally and you can reload its config def initself raise environmenterror autotokenizer is designed to be instantiated using the autotokenizer frompretrainedpretrainedmodelnameorpath method classmethod replacelistoptionindocstringstokenizermappingnames def frompretrainedcls pretrainedmodelnameorpath inputs kwargs r instantiate one of the tokenizer classes of the library from a pretrained model vocabulary the tokenizer class to instantiate is selected based on the modeltype property of the config object either passed as an argument or loaded from pretrainedmodelnameorpath if possible or when it s missing by falling back to using pattern matching on pretrainedmodelnameorpath list options params pretrainedmodelnameorpath str or os pathlike can be either a string the model id of a predefined tokenizer hosted inside a model repo on huggingface co valid model ids can be located at the rootlevel like bertbaseuncased or namespaced under a user or organization name like dbmdzbertbasegermancased a path to a directory containing vocabulary files required by the tokenizer for instance saved using the pretrainedtokenizer savepretrained method e g mymodeldirectory a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file like bert or xlnet e g mymodeldirectoryvocab txt not applicable to all derived classes inputs additional positional arguments optional will be passed along to the tokenizer init method config pretrainedconfig optional the configuration object used to determine the tokenizer class to instantiate cachedir str or os pathlike optional path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used forcedownload bool optional defaults to false whether or not to force the redownload the model weights and configuration files and override the cached versions if they exist resumedownload bool optional defaults to false whether or not to delete incompletely received files will attempt to resume the download if such a file exists proxies dictstr str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a gitbased system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git subfolder str optional in case the relevant files are located inside a subfolder of the model repo on huggingface co e g for facebookragtokenbase specify it here usefast bool optional defaults to true use a fast rustbased tokenizerhttps huggingface codocstokenizersindex if it is supported for a given model if a fast tokenizer is not available for a given model a normal pythonbased tokenizer is returned instead tokenizertype str optional tokenizer type to be loaded trustremotecode bool optional defaults to false whether or not to allow for custom models defined on the hub in their own modeling files this option should only be set to true for repositories you trust and in which you have read the code as it will execute code present on the hub on your local machine kwargs additional keyword arguments optional will be passed to the tokenizer init method can be used to set special tokens like bostoken eostoken unktoken septoken padtoken clstoken masktoken additionalspecialtokens see parameters in the init for more details examples python from transformers import autotokenizer download vocabulary from huggingface co and cache tokenizer autotokenizer frompretrainedbertbaseuncased download vocabulary from huggingface co useruploaded and cache tokenizer autotokenizer frompretraineddbmdzbertbasegermancased if vocabulary files are in a directory e g tokenizer was saved using savepretrained testsavedmodel tokenizer autotokenizer frompretrained testbertsavedmodel download vocabulary from huggingface co and define modelspecific arguments tokenizer autotokenizer frompretrainedrobertabase addprefixspacetrue useauthtoken kwargs popuseauthtoken none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if kwargs gettoken none is not none raise valueerror token and useauthtoken are both specified please set only the argument token kwargstoken useauthtoken config kwargs popconfig none kwargsfromauto true usefast kwargs popusefast true tokenizertype kwargs poptokenizertype none trustremotecode kwargs poptrustremotecode none first let s see whether the tokenizertype is passed so that we can leverage it if tokenizertype is not none tokenizerclass none tokenizerclasstuple tokenizermappingnames gettokenizertype none if tokenizerclasstuple is none raise valueerror fpassed tokenizertype tokenizertype does not exist tokenizertype should be one of f joinc for c in tokenizermappingnames keys tokenizerclassname tokenizerfastclassname tokenizerclasstuple if usefast if tokenizerfastclassname is not none tokenizerclass tokenizerclassfromnametokenizerfastclassname else logger warning usefast is set to true but the tokenizer class does not have a fast version falling back to the slow version if tokenizerclass is none tokenizerclass tokenizerclassfromnametokenizerclassname if tokenizerclass is none raise valueerrorftokenizer class tokenizerclassname is not currently imported return tokenizerclass frompretrainedpretrainedmodelnameorpath inputs kwargs next let s try to use the tokenizerconfig file to get the tokenizer class tokenizerconfig gettokenizerconfigpretrainedmodelnameorpath kwargs if commithash in tokenizerconfig kwargscommithash tokenizerconfigcommithash configtokenizerclass tokenizerconfig gettokenizerclass tokenizerautomap none if automap in tokenizerconfig if isinstancetokenizerconfigautomap tuple list legacy format for dynamic tokenizers tokenizerautomap tokenizerconfigautomap else tokenizerautomap tokenizerconfigautomap getautotokenizer none if that did not work let s try to use the config if configtokenizerclass is none if not isinstanceconfig pretrainedconfig config autoconfig frompretrained pretrainedmodelnameorpath trustremotecodetrustremotecode kwargs configtokenizerclass config tokenizerclass if hasattrconfig automap and autotokenizer in config automap tokenizerautomap config automapautotokenizer hasremotecode tokenizerautomap is not none haslocalcode configtokenizerclass is not none or typeconfig in tokenizermapping trustremotecode resolvetrustremotecode trustremotecode pretrainedmodelnameorpath haslocalcode hasremotecode if hasremotecode and trustremotecode if usefast and tokenizerautomap1 is not none classref tokenizerautomap1 else classref tokenizerautomap0 tokenizerclass getclassfromdynamicmoduleclassref pretrainedmodelnameorpath kwargs kwargs popcoderevision none if os path isdirpretrainedmodelnameorpath tokenizerclass registerforautoclass return tokenizerclass frompretrainedpretrainedmodelnameorpath inputs kwargs elif configtokenizerclass is not none tokenizerclass none if usefast and not configtokenizerclass endswithfast tokenizerclasscandidate fconfigtokenizerclassfast tokenizerclass tokenizerclassfromnametokenizerclasscandidate if tokenizerclass is none tokenizerclasscandidate configtokenizerclass tokenizerclass tokenizerclassfromnametokenizerclasscandidate if tokenizerclass is none raise valueerror ftokenizer class tokenizerclasscandidate does not exist or is not currently imported return tokenizerclass frompretrainedpretrainedmodelnameorpath inputs kwargs otherwise we have to be creative if model is an encoder decoder the encoder tokenizer class is used by default if isinstanceconfig encoderdecoderconfig if typeconfig decoder is not typeconfig encoder noqa e721 logger warning fthe encoder model config class config encoder class is different from the decoder model fconfig class config decoder class it is not recommended to use the autotokenizer frompretrained method in this case please use the encoder and decoder specific tokenizer classes config config encoder modeltype configclasstomodeltypetypeconfig name if modeltype is not none tokenizerclasspy tokenizerclassfast tokenizermappingtypeconfig if tokenizerclassfast and usefast or tokenizerclasspy is none return tokenizerclassfast frompretrainedpretrainedmodelnameorpath inputs kwargs else if tokenizerclasspy is not none return tokenizerclasspy frompretrainedpretrainedmodelnameorpath inputs kwargs else raise valueerror this tokenizer cannot be instantiated please make sure you have sentencepiece installed in order to use this tokenizer raise valueerror funrecognized configuration class config class to build an autotokenizer n fmodel type should be one of joinc name for c in tokenizermapping keys def registerconfigclass slowtokenizerclassnone fasttokenizerclassnone existokfalse if slowtokenizerclass is none and fasttokenizerclass is none raise valueerroryou need to pass either a slowtokenizerclass or a fasttokenizerclass if slowtokenizerclass is not none and issubclassslowtokenizerclass pretrainedtokenizerfast raise valueerroryou passed a fast tokenizer in the slowtokenizerclass if fasttokenizerclass is not none and issubclassfasttokenizerclass pretrainedtokenizer raise valueerroryou passed a slow tokenizer in the fasttokenizerclass if slowtokenizerclass is not none and fasttokenizerclass is not none and issubclassfasttokenizerclass pretrainedtokenizerfast and fasttokenizerclass slowtokenizerclass slowtokenizerclass raise valueerror the fast tokenizer class you are passing has a slowtokenizerclass attribute that is not consistent with the slow tokenizer class you passed fast tokenizer has ffasttokenizerclass slowtokenizerclass and you passed slowtokenizerclass fix one of those so they match avoid resetting a set slowfast tokenizer if we are passing just the other ones if configclass in tokenizermapping extracontent existingslow existingfast tokenizermappingconfigclass if slowtokenizerclass is none slowtokenizerclass existingslow if fasttokenizerclass is none fasttokenizerclass existingfast tokenizermapping registerconfigclass slowtokenizerclass fasttokenizerclass existokexistok coding utf 8 2018 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license auto tokenizer class this significantly improves completion suggestion performance when the transformers package is used with microsoft s pylance language server we did not fine the class but maybe it s because a dep is missing in that case the class will be in the main init and we return the proper dummy to get an appropriate error message loads the tokenizer configuration from a pretrained model tokenizer configuration args pretrained_model_name_or_path str or os pathlike this can be either a string the model id of a pretrained model configuration hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing a configuration file saved using the pretrainedtokenizer save_pretrained method e g my_model_directory cache_dir str or os pathlike optional path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used force_download bool optional defaults to false whether or not to force to re download the configuration files and override the cached versions if they exist resume_download bool optional defaults to false whether or not to delete incompletely received file attempts to resume the download if such a file exists proxies dict str str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request token str or bool optional the token to use as http bearer ization for remote files if true will use the token generated when running huggingface cli login stored in huggingface revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a git based system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git local_files_only bool optional defaults to false if true will only try to load the tokenizer configuration from local files subfolder str optional defaults to in case the tokenizer config is located inside a subfolder of the model repo on huggingface co you can specify the folder name here tip passing token true is required when you want to use a private model tip returns dict the configuration of the tokenizer examples python download configuration from huggingface co and cache tokenizer_config get_tokenizer_config bert base uncased this model does not have a tokenizer config so the result will be an empty dict tokenizer_config get_tokenizer_config xlm roberta base save a pretrained tokenizer locally and you can reload its config from transformers import autotokenizer tokenizer autotokenizer from_pretrained bert base cased tokenizer save_pretrained tokenizer test tokenizer_config get_tokenizer_config tokenizer test this is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the autotokenizer from_pretrained class method this class cannot be instantiated directly using __init__ throws an error instantiate one of the tokenizer classes of the library from a pretrained model vocabulary the tokenizer class to instantiate is selected based on the model_type property of the config object either passed as an argument or loaded from pretrained_model_name_or_path if possible or when it s missing by falling back to using pattern matching on pretrained_model_name_or_path list options params pretrained_model_name_or_path str or os pathlike can be either a string the model id of a predefined tokenizer hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing vocabulary files required by the tokenizer for instance saved using the pretrainedtokenizer save_pretrained method e g my_model_directory a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file like bert or xlnet e g my_model_directory vocab txt not applicable to all derived classes inputs additional positional arguments optional will be passed along to the tokenizer __init__ method config pretrainedconfig optional the configuration object used to determine the tokenizer class to instantiate cache_dir str or os pathlike optional path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used force_download bool optional defaults to false whether or not to force the re download the model weights and configuration files and override the cached versions if they exist resume_download bool optional defaults to false whether or not to delete incompletely received files will attempt to resume the download if such a file exists proxies dict str str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a git based system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git subfolder str optional in case the relevant files are located inside a subfolder of the model repo on huggingface co e g for facebook rag token base specify it here use_fast bool optional defaults to true use a fast rust based tokenizer https huggingface co docs tokenizers index if it is supported for a given model if a fast tokenizer is not available for a given model a normal python based tokenizer is returned instead tokenizer_type str optional tokenizer type to be loaded trust_remote_code bool optional defaults to false whether or not to allow for custom models defined on the hub in their own modeling files this option should only be set to true for repositories you trust and in which you have read the code as it will execute code present on the hub on your local machine kwargs additional keyword arguments optional will be passed to the tokenizer __init__ method can be used to set special tokens like bos_token eos_token unk_token sep_token pad_token cls_token mask_token additional_special_tokens see parameters in the __init__ for more details examples python from transformers import autotokenizer download vocabulary from huggingface co and cache tokenizer autotokenizer from_pretrained bert base uncased download vocabulary from huggingface co user uploaded and cache tokenizer autotokenizer from_pretrained dbmdz bert base german cased if vocabulary files are in a directory e g tokenizer was saved using save_pretrained test saved_model tokenizer autotokenizer from_pretrained test bert_saved_model download vocabulary from huggingface co and define model specific arguments tokenizer autotokenizer from_pretrained roberta base add_prefix_space true first let s see whether the tokenizer_type is passed so that we can leverage it next let s try to use the tokenizer_config file to get the tokenizer class legacy format for dynamic tokenizers if that did not work let s try to use the config otherwise we have to be creative if model is an encoder decoder the encoder tokenizer class is used by default noqa e721 register a new tokenizer in this mapping args config_class pretrainedconfig the configuration corresponding to the model to register slow_tokenizer_class pretrainedtokenizer optional the slow tokenizer to register fast_tokenizer_class pretrainedtokenizerfast optional the fast tokenizer to register avoid resetting a set slow fast tokenizer if we are passing just the other ones
import importlib
import json
import os
import warnings
from collections import OrderedDict
from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union

from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
from ...utils import cached_file, extract_commit_hash, is_sentencepiece_available, is_tokenizers_available, logging
from ..encoder_decoder import EncoderDecoderConfig
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    config_class_to_model_type,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


if is_tokenizers_available():
    from ...tokenization_utils_fast import PreTrainedTokenizerFast
else:
    PreTrainedTokenizerFast = None


logger = logging.get_logger(__name__)

if TYPE_CHECKING:
    
    
    TOKENIZER_MAPPING_NAMES: OrderedDict[str, Tuple[Optional[str], Optional[str]]] = OrderedDict()
else:
    TOKENIZER_MAPPING_NAMES = OrderedDict(
        [
            (
                "albert",
                (
                    "AlbertTokenizer" if is_sentencepiece_available() else None,
                    "AlbertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("align", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("bark", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("bart", ("BartTokenizer", "BartTokenizerFast")),
            (
                "barthez",
                (
                    "BarthezTokenizer" if is_sentencepiece_available() else None,
                    "BarthezTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("bartpho", ("BartphoTokenizer", None)),
            ("bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("bert-generation", ("BertGenerationTokenizer" if is_sentencepiece_available() else None, None)),
            ("bert-japanese", ("BertJapaneseTokenizer", None)),
            ("bertweet", ("BertweetTokenizer", None)),
            (
                "big_bird",
                (
                    "BigBirdTokenizer" if is_sentencepiece_available() else None,
                    "BigBirdTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("bigbird_pegasus", ("PegasusTokenizer", "PegasusTokenizerFast" if is_tokenizers_available() else None)),
            ("biogpt", ("BioGptTokenizer", None)),
            ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")),
            ("blenderbot-small", ("BlenderbotSmallTokenizer", None)),
            ("blip", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("blip-2", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("bloom", (None, "BloomTokenizerFast" if is_tokenizers_available() else None)),
            ("bridgetower", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            ("bros", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("byt5", ("ByT5Tokenizer", None)),
            (
                "camembert",
                (
                    "CamembertTokenizer" if is_sentencepiece_available() else None,
                    "CamembertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("canine", ("CanineTokenizer", None)),
            ("chinese_clip", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            (
                "clap",
                (
                    "RobertaTokenizer",
                    "RobertaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "clip",
                (
                    "CLIPTokenizer",
                    "CLIPTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "clipseg",
                (
                    "CLIPTokenizer",
                    "CLIPTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("clvp", ("ClvpTokenizer", None)),
            (
                "code_llama",
                (
                    "CodeLlamaTokenizer" if is_sentencepiece_available() else None,
                    "CodeLlamaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("codegen", ("CodeGenTokenizer", "CodeGenTokenizerFast" if is_tokenizers_available() else None)),
            ("convbert", ("ConvBertTokenizer", "ConvBertTokenizerFast" if is_tokenizers_available() else None)),
            (
                "cpm",
                (
                    "CpmTokenizer" if is_sentencepiece_available() else None,
                    "CpmTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("cpmant", ("CpmAntTokenizer", None)),
            ("ctrl", ("CTRLTokenizer", None)),
            ("data2vec-audio", ("Wav2Vec2CTCTokenizer", None)),
            ("data2vec-text", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            ("deberta", ("DebertaTokenizer", "DebertaTokenizerFast" if is_tokenizers_available() else None)),
            (
                "deberta-v2",
                (
                    "DebertaV2Tokenizer" if is_sentencepiece_available() else None,
                    "DebertaV2TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("distilbert", ("DistilBertTokenizer", "DistilBertTokenizerFast" if is_tokenizers_available() else None)),
            (
                "dpr",
                (
                    "DPRQuestionEncoderTokenizer",
                    "DPRQuestionEncoderTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("electra", ("ElectraTokenizer", "ElectraTokenizerFast" if is_tokenizers_available() else None)),
            ("ernie", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("ernie_m", ("ErnieMTokenizer" if is_sentencepiece_available() else None, None)),
            ("esm", ("EsmTokenizer", None)),
            ("falcon", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
            ("flaubert", ("FlaubertTokenizer", None)),
            ("fnet", ("FNetTokenizer", "FNetTokenizerFast" if is_tokenizers_available() else None)),
            ("fsmt", ("FSMTTokenizer", None)),
            ("funnel", ("FunnelTokenizer", "FunnelTokenizerFast" if is_tokenizers_available() else None)),
            ("git", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("gpt-sw3", ("GPTSw3Tokenizer" if is_sentencepiece_available() else None, None)),
            ("gpt2", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("gpt_bigcode", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("gpt_neo", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("gpt_neox", (None, "GPTNeoXTokenizerFast" if is_tokenizers_available() else None)),
            ("gpt_neox_japanese", ("GPTNeoXJapaneseTokenizer", None)),
            ("gptj", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("gptsan-japanese", ("GPTSanJapaneseTokenizer", None)),
            ("groupvit", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
            ("herbert", ("HerbertTokenizer", "HerbertTokenizerFast" if is_tokenizers_available() else None)),
            ("hubert", ("Wav2Vec2CTCTokenizer", None)),
            ("ibert", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            ("idefics", (None, "LlamaTokenizerFast" if is_tokenizers_available() else None)),
            ("instructblip", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("jukebox", ("JukeboxTokenizer", None)),
            (
                "kosmos-2",
                (
                    "XLMRobertaTokenizer" if is_sentencepiece_available() else None,
                    "XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("layoutlm", ("LayoutLMTokenizer", "LayoutLMTokenizerFast" if is_tokenizers_available() else None)),
            ("layoutlmv2", ("LayoutLMv2Tokenizer", "LayoutLMv2TokenizerFast" if is_tokenizers_available() else None)),
            ("layoutlmv3", ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast" if is_tokenizers_available() else None)),
            ("layoutxlm", ("LayoutXLMTokenizer", "LayoutXLMTokenizerFast" if is_tokenizers_available() else None)),
            ("led", ("LEDTokenizer", "LEDTokenizerFast" if is_tokenizers_available() else None)),
            ("lilt", ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast" if is_tokenizers_available() else None)),
            (
                "llama",
                (
                    "LlamaTokenizer" if is_sentencepiece_available() else None,
                    "LlamaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)),
            (
                "longt5",
                (
                    "T5Tokenizer" if is_sentencepiece_available() else None,
                    "T5TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("luke", ("LukeTokenizer", None)),
            ("lxmert", ("LxmertTokenizer", "LxmertTokenizerFast" if is_tokenizers_available() else None)),
            ("m2m_100", ("M2M100Tokenizer" if is_sentencepiece_available() else None, None)),
            ("marian", ("MarianTokenizer" if is_sentencepiece_available() else None, None)),
            (
                "mbart",
                (
                    "MBartTokenizer" if is_sentencepiece_available() else None,
                    "MBartTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "mbart50",
                (
                    "MBart50Tokenizer" if is_sentencepiece_available() else None,
                    "MBart50TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("mega", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("mgp-str", ("MgpstrTokenizer", None)),
            (
                "mistral",
                (
                    "LlamaTokenizer" if is_sentencepiece_available() else None,
                    "LlamaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("mluke", ("MLukeTokenizer" if is_sentencepiece_available() else None, None)),
            ("mobilebert", ("MobileBertTokenizer", "MobileBertTokenizerFast" if is_tokenizers_available() else None)),
            ("mpnet", ("MPNetTokenizer", "MPNetTokenizerFast" if is_tokenizers_available() else None)),
            ("mpt", (None, "GPTNeoXTokenizerFast" if is_tokenizers_available() else None)),
            ("mra", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            (
                "mt5",
                (
                    "MT5Tokenizer" if is_sentencepiece_available() else None,
                    "MT5TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("musicgen", ("T5Tokenizer", "T5TokenizerFast" if is_tokenizers_available() else None)),
            ("mvp", ("MvpTokenizer", "MvpTokenizerFast" if is_tokenizers_available() else None)),
            ("nezha", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            (
                "nllb",
                (
                    "NllbTokenizer" if is_sentencepiece_available() else None,
                    "NllbTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "nllb-moe",
                (
                    "NllbTokenizer" if is_sentencepiece_available() else None,
                    "NllbTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "nystromformer",
                (
                    "AlbertTokenizer" if is_sentencepiece_available() else None,
                    "AlbertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("oneformer", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
            ("openai-gpt", ("OpenAIGPTTokenizer", "OpenAIGPTTokenizerFast" if is_tokenizers_available() else None)),
            ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)),
            ("owlv2", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
            ("owlvit", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
            (
                "pegasus",
                (
                    "PegasusTokenizer" if is_sentencepiece_available() else None,
                    "PegasusTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "pegasus_x",
                (
                    "PegasusTokenizer" if is_sentencepiece_available() else None,
                    "PegasusTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "perceiver",
                (
                    "PerceiverTokenizer",
                    None,
                ),
            ),
            (
                "persimmon",
                (
                    "LlamaTokenizer" if is_sentencepiece_available() else None,
                    "LlamaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("phi", ("CodeGenTokenizer", "CodeGenTokenizerFast" if is_tokenizers_available() else None)),
            ("phobert", ("PhobertTokenizer", None)),
            ("pix2struct", ("T5Tokenizer", "T5TokenizerFast" if is_tokenizers_available() else None)),
            ("plbart", ("PLBartTokenizer" if is_sentencepiece_available() else None, None)),
            ("prophetnet", ("ProphetNetTokenizer", None)),
            ("qdqbert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("rag", ("RagTokenizer", None)),
            ("realm", ("RealmTokenizer", "RealmTokenizerFast" if is_tokenizers_available() else None)),
            (
                "reformer",
                (
                    "ReformerTokenizer" if is_sentencepiece_available() else None,
                    "ReformerTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "rembert",
                (
                    "RemBertTokenizer" if is_sentencepiece_available() else None,
                    "RemBertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("retribert", ("RetriBertTokenizer", "RetriBertTokenizerFast" if is_tokenizers_available() else None)),
            ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)),
            (
                "roberta-prelayernorm",
                ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None),
            ),
            ("roc_bert", ("RoCBertTokenizer", None)),
            ("roformer", ("RoFormerTokenizer", "RoFormerTokenizerFast" if is_tokenizers_available() else None)),
            ("rwkv", (None, "GPTNeoXTokenizerFast" if is_tokenizers_available() else None)),
            (
                "seamless_m4t",
                (
                    "SeamlessM4TTokenizer" if is_sentencepiece_available() else None,
                    "SeamlessM4TTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "seamless_m4t_v2",
                (
                    "SeamlessM4TTokenizer" if is_sentencepiece_available() else None,
                    "SeamlessM4TTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)),
            ("speech_to_text_2", ("Speech2Text2Tokenizer", None)),
            ("speecht5", ("SpeechT5Tokenizer" if is_sentencepiece_available() else None, None)),
            ("splinter", ("SplinterTokenizer", "SplinterTokenizerFast")),
            (
                "squeezebert",
                ("SqueezeBertTokenizer", "SqueezeBertTokenizerFast" if is_tokenizers_available() else None),
            ),
            (
                "switch_transformers",
                (
                    "T5Tokenizer" if is_sentencepiece_available() else None,
                    "T5TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "t5",
                (
                    "T5Tokenizer" if is_sentencepiece_available() else None,
                    "T5TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("tapas", ("TapasTokenizer", None)),
            ("tapex", ("TapexTokenizer", None)),
            ("transfo-xl", ("TransfoXLTokenizer", None)),
            ("tvp", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            (
                "umt5",
                (
                    "T5Tokenizer" if is_sentencepiece_available() else None,
                    "T5TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("vilt", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("visual_bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
            ("vits", ("VitsTokenizer", None)),
            ("wav2vec2", ("Wav2Vec2CTCTokenizer", None)),
            ("wav2vec2-conformer", ("Wav2Vec2CTCTokenizer", None)),
            ("wav2vec2_phoneme", ("Wav2Vec2PhonemeCTCTokenizer", None)),
            ("whisper", ("WhisperTokenizer", "WhisperTokenizerFast" if is_tokenizers_available() else None)),
            ("xclip", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
            (
                "xglm",
                (
                    "XGLMTokenizer" if is_sentencepiece_available() else None,
                    "XGLMTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            ("xlm", ("XLMTokenizer", None)),
            ("xlm-prophetnet", ("XLMProphetNetTokenizer" if is_sentencepiece_available() else None, None)),
            (
                "xlm-roberta",
                (
                    "XLMRobertaTokenizer" if is_sentencepiece_available() else None,
                    "XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "xlm-roberta-xl",
                (
                    "XLMRobertaTokenizer" if is_sentencepiece_available() else None,
                    "XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "xlnet",
                (
                    "XLNetTokenizer" if is_sentencepiece_available() else None,
                    "XLNetTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "xmod",
                (
                    "XLMRobertaTokenizer" if is_sentencepiece_available() else None,
                    "XLMRobertaTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
            (
                "yoso",
                (
                    "AlbertTokenizer" if is_sentencepiece_available() else None,
                    "AlbertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
        ]
    )

TOKENIZER_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, TOKENIZER_MAPPING_NAMES)

CONFIG_TO_TYPE = {v: k for k, v in CONFIG_MAPPING_NAMES.items()}


def tokenizer_class_from_name(class_name: str):
    if class_name == "PreTrainedTokenizerFast":
        return PreTrainedTokenizerFast

    for module_name, tokenizers in TOKENIZER_MAPPING_NAMES.items():
        if class_name in tokenizers:
            module_name = model_type_to_module_name(module_name)

            module = importlib.import_module(f".{module_name}", "transformers.models")
            try:
                return getattr(module, class_name)
            except AttributeError:
                continue

    for config, tokenizers in TOKENIZER_MAPPING._extra_content.items():
        for tokenizer in tokenizers:
            if getattr(tokenizer, "__name__", None) == class_name:
                return tokenizer

    
    
    main_module = importlib.import_module("transformers")
    if hasattr(main_module, class_name):
        return getattr(main_module, class_name)

    return None


def get_tokenizer_config(
    pretrained_model_name_or_path: Union[str, os.PathLike],
    cache_dir: Optional[Union[str, os.PathLike]] = None,
    force_download: bool = False,
    resume_download: bool = False,
    proxies: Optional[Dict[str, str]] = None,
    token: Optional[Union[bool, str]] = None,
    revision: Optional[str] = None,
    local_files_only: bool = False,
    subfolder: str = "",
    **kwargs,
):
    
    use_auth_token = kwargs.pop("use_auth_token", None)
    if use_auth_token is not None:
        warnings.warn(
            "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
            FutureWarning,
        )
        if token is not None:
            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
        token = use_auth_token

    commit_hash = kwargs.get("_commit_hash", None)
    resolved_config_file = cached_file(
        pretrained_model_name_or_path,
        TOKENIZER_CONFIG_FILE,
        cache_dir=cache_dir,
        force_download=force_download,
        resume_download=resume_download,
        proxies=proxies,
        token=token,
        revision=revision,
        local_files_only=local_files_only,
        subfolder=subfolder,
        _raise_exceptions_for_missing_entries=False,
        _raise_exceptions_for_connection_errors=False,
        _commit_hash=commit_hash,
    )
    if resolved_config_file is None:
        logger.info("Could not locate the tokenizer configuration file, will try to use the model config instead.")
        return {}
    commit_hash = extract_commit_hash(resolved_config_file, commit_hash)

    with open(resolved_config_file, encoding="utf-8") as reader:
        result = json.load(reader)
    result["_commit_hash"] = commit_hash
    return result


class AutoTokenizer:
    r

    def __init__(self):
        raise EnvironmentError(
            "AutoTokenizer is designed to be instantiated "
            "using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method."
        )

    @classmethod
    @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
    def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
        r
        use_auth_token = kwargs.pop("use_auth_token", None)
        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if kwargs.get("token", None) is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            kwargs["token"] = use_auth_token

        config = kwargs.pop("config", None)
        kwargs["_from_auto"] = True

        use_fast = kwargs.pop("use_fast", True)
        tokenizer_type = kwargs.pop("tokenizer_type", None)
        trust_remote_code = kwargs.pop("trust_remote_code", None)

        
        if tokenizer_type is not None:
            tokenizer_class = None
            tokenizer_class_tuple = TOKENIZER_MAPPING_NAMES.get(tokenizer_type, None)

            if tokenizer_class_tuple is None:
                raise ValueError(
                    f"Passed `tokenizer_type` {tokenizer_type} does not exist. `tokenizer_type` should be one of "
                    f"{', '.join(c for c in TOKENIZER_MAPPING_NAMES.keys())}."
                )

            tokenizer_class_name, tokenizer_fast_class_name = tokenizer_class_tuple

            if use_fast:
                if tokenizer_fast_class_name is not None:
                    tokenizer_class = tokenizer_class_from_name(tokenizer_fast_class_name)
                else:
                    logger.warning(
                        "`use_fast` is set to `True` but the tokenizer class does not have a fast version. "
                        " Falling back to the slow version."
                    )
            if tokenizer_class is None:
                tokenizer_class = tokenizer_class_from_name(tokenizer_class_name)

            if tokenizer_class is None:
                raise ValueError(f"Tokenizer class {tokenizer_class_name} is not currently imported.")

            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        
        tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
        if "_commit_hash" in tokenizer_config:
            kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]
        config_tokenizer_class = tokenizer_config.get("tokenizer_class")
        tokenizer_auto_map = None
        if "auto_map" in tokenizer_config:
            if isinstance(tokenizer_config["auto_map"], (tuple, list)):
                
                tokenizer_auto_map = tokenizer_config["auto_map"]
            else:
                tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)

        
        if config_tokenizer_class is None:
            if not isinstance(config, PretrainedConfig):
                config = AutoConfig.from_pretrained(
                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
                )
            config_tokenizer_class = config.tokenizer_class
            if hasattr(config, "auto_map") and "AutoTokenizer" in config.auto_map:
                tokenizer_auto_map = config.auto_map["AutoTokenizer"]

        has_remote_code = tokenizer_auto_map is not None
        has_local_code = config_tokenizer_class is not None or type(config) in TOKENIZER_MAPPING
        trust_remote_code = resolve_trust_remote_code(
            trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code
        )

        if has_remote_code and trust_remote_code:
            if use_fast and tokenizer_auto_map[1] is not None:
                class_ref = tokenizer_auto_map[1]
            else:
                class_ref = tokenizer_auto_map[0]
            tokenizer_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
            _ = kwargs.pop("code_revision", None)
            if os.path.isdir(pretrained_model_name_or_path):
                tokenizer_class.register_for_auto_class()
            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
        elif config_tokenizer_class is not None:
            tokenizer_class = None
            if use_fast and not config_tokenizer_class.endswith("Fast"):
                tokenizer_class_candidate = f"{config_tokenizer_class}Fast"
                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
            if tokenizer_class is None:
                tokenizer_class_candidate = config_tokenizer_class
                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
            if tokenizer_class is None:
                raise ValueError(
                    f"Tokenizer class {tokenizer_class_candidate} does not exist or is not currently imported."
                )
            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        
        
        if isinstance(config, EncoderDecoderConfig):
            if type(config.decoder) is not type(config.encoder):  
                logger.warning(
                    f"The encoder model config class: {config.encoder.__class__} is different from the decoder model "
                    f"config class: {config.decoder.__class__}. It is not recommended to use the "
                    "`AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder "
                    "specific tokenizer classes."
                )
            config = config.encoder

        model_type = config_class_to_model_type(type(config).__name__)
        if model_type is not None:
            tokenizer_class_py, tokenizer_class_fast = TOKENIZER_MAPPING[type(config)]
            if tokenizer_class_fast and (use_fast or tokenizer_class_py is None):
                return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
            else:
                if tokenizer_class_py is not None:
                    return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
                else:
                    raise ValueError(
                        "This tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed "
                        "in order to use this tokenizer."
                    )

        raise ValueError(
            f"Unrecognized configuration class {config.__class__} to build an AutoTokenizer.\n"
            f"Model type should be one of {', '.join(c.__name__ for c in TOKENIZER_MAPPING.keys())}."
        )

    def register(config_class, slow_tokenizer_class=None, fast_tokenizer_class=None, exist_ok=False):
        
        if slow_tokenizer_class is None and fast_tokenizer_class is None:
            raise ValueError("You need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_class")
        if slow_tokenizer_class is not None and issubclass(slow_tokenizer_class, PreTrainedTokenizerFast):
            raise ValueError("You passed a fast tokenizer in the `slow_tokenizer_class`.")
        if fast_tokenizer_class is not None and issubclass(fast_tokenizer_class, PreTrainedTokenizer):
            raise ValueError("You passed a slow tokenizer in the `fast_tokenizer_class`.")

        if (
            slow_tokenizer_class is not None
            and fast_tokenizer_class is not None
            and issubclass(fast_tokenizer_class, PreTrainedTokenizerFast)
            and fast_tokenizer_class.slow_tokenizer_class != slow_tokenizer_class
        ):
            raise ValueError(
                "The fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not "
                "consistent with the slow tokenizer class you passed (fast tokenizer has "
                f"{fast_tokenizer_class.slow_tokenizer_class} and you passed {slow_tokenizer_class}. Fix one of those "
                "so they match!"
            )

        
        if config_class in TOKENIZER_MAPPING._extra_content:
            existing_slow, existing_fast = TOKENIZER_MAPPING[config_class]
            if slow_tokenizer_class is None:
                slow_tokenizer_class = existing_slow
            if fast_tokenizer_class is None:
                fast_tokenizer_class = existing_fast

        TOKENIZER_MAPPING.register(config_class, (slow_tokenizer_class, fast_tokenizer_class), exist_ok=exist_ok)