codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license generation configuration class and utilities import copy import json import os import warnings from typing import any dict optional union from import version from configurationutils import pretrainedconfig from utils import generationconfigname pushtohubmixin cachedfile downloadurl extractcommithash isremoteurl logging logger logging getloggername metadatafields frommodelconfig commithash originalobjecthash transformersversion class generationconfigpushtohubmixin noformat r class that holds a configuration for a generation task a generate call supports the following generation methods for textdecoder texttotext speechtotext and visiontotext models greedy decoding by calling generation generationmixin greedysearch if numbeams1 and dosamplefalse contrastive search by calling generation generationmixin contrastivesearch if penaltyalpha0 and topk1 multinomial sampling by calling generation generationmixin sample if numbeams1 and dosampletrue beamsearch decoding by calling generation generationmixin beamsearch if numbeams1 and dosamplefalse beamsearch multinomial sampling by calling generation generationmixin beamsample if numbeams1 and dosampletrue diverse beamsearch decoding by calling generation generationmixin groupbeamsearch if numbeams1 and numbeamgroups1 constrained beamsearch decoding by calling generation generationmixin constrainedbeamsearch if constraints none or forcewordsids none assisted decoding by calling generation generationmixin assisteddecoding if assistantmodel is passed to generate you do not need to call any of the above methods directly pass custom parameter values to generate to learn more about decoding strategies refer to the text generation strategies guide generationstrategies arg parameters that control the length of the output maxlength int optional defaults to 20 the maximum length the generated tokens can have corresponds to the length of the input prompt maxnewtokens its effect is overridden by maxnewtokens if also set maxnewtokens int optional the maximum numbers of tokens to generate ignoring the number of tokens in the prompt minlength int optional defaults to 0 the minimum length of the sequence to be generated corresponds to the length of the input prompt minnewtokens its effect is overridden by minnewtokens if also set minnewtokens int optional the minimum numbers of tokens to generate ignoring the number of tokens in the prompt earlystopping bool or str optional defaults to false controls the stopping condition for beambased methods like beamsearch it accepts the following values true where the generation stops as soon as there are numbeams complete candidates false where an heuristic is applied and the generation stops when is it very unlikely to find better candidates never where the beam search procedure only stops when there cannot be better candidates canonical beam search algorithm maxtimefloat optional the maximum amount of time you allow the computation to run for in seconds generation will still finish the current pass after allocated time has been passed parameters that control the generation strategy used dosample bool optional defaults to false whether or not to use sampling use greedy decoding otherwise numbeams int optional defaults to 1 number of beams for beam search 1 means no beam search numbeamgroups int optional defaults to 1 number of groups to divide numbeams into in order to ensure diversity among different groups of beams this paperhttps arxiv orgpdf1610 02424 pdf for more details penaltyalpha float optional the values balance the model confidence and the degeneration penalty in contrastive search decoding usecache bool optional defaults to true whether or not the model should use the past last keyvalues attentions if applicable to the model to speed up decoding parameters for manipulation of the model output logits temperature float optional defaults to 1 0 the value used to modulate the next token probabilities topk int optional defaults to 50 the number of highest probability vocabulary tokens to keep for topkfiltering topp float optional defaults to 1 0 if set to float 1 only the smallest set of most probable tokens with probabilities that add up to topp or higher are kept for generation typicalp float optional defaults to 1 0 local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next given the partial text already generated if set to float 1 the smallest set of the most locally typical tokens with probabilities that add up to typicalp or higher are kept for generation see this paperhttps arxiv orgpdf2202 00666 pdf for more details epsiloncutoff float optional defaults to 0 0 if set to float strictly between 0 and 1 only tokens with a conditional probability greater than epsiloncutoff will be sampled in the paper suggested values range from 3e4 to 9e4 depending on the size of the model see truncation sampling as language model desmoothinghttps arxiv orgabs2210 15191 for more details etacutoff float optional defaults to 0 0 eta sampling is a hybrid of locally typical sampling and epsilon sampling if set to float strictly between 0 and 1 a token is only considered if it is greater than either etacutoff or sqrtetacutoff expentropysoftmaxnexttokenlogits the latter term is intuitively the expected next token probability scaled by sqrtetacutoff in the paper suggested values range from 3e4 to 2e3 depending on the size of the model see truncation sampling as language model desmoothinghttps arxiv orgabs2210 15191 for more details diversitypenalty float optional defaults to 0 0 this value is subtracted from a beam s score if it generates a token same as any beam from other group at a particular time note that diversitypenalty is only effective if group beam search is enabled repetitionpenalty float optional defaults to 1 0 the parameter for repetition penalty 1 0 means no penalty see this paperhttps arxiv orgpdf1909 05858 pdf for more details encoderrepetitionpenalty float optional defaults to 1 0 the paramater for encoderrepetitionpenalty an exponential penalty on sequences that are not in the original input 1 0 means no penalty lengthpenalty float optional defaults to 1 0 exponential penalty to the length that is used with beambased generation it is applied as an exponent to the sequence length which in turn is used to divide the score of the sequence since the score is the log likelihood of the sequence i e negative lengthpenalty 0 0 promotes longer sequences while lengthpenalty 0 0 encourages shorter sequences norepeatngramsize int optional defaults to 0 if set to int 0 all ngrams of that size can only occur once badwordsidslistlistint optional list of list of token ids that are not allowed to be generated check generation nobadwordslogitsprocessor for further documentation and examples forcewordsidslistlistint or listlistlistint optional list of token ids that must be generated if given a listlistint this is treated as a simple list of words that must be included the opposite to badwordsids if given listlistlistint this triggers a disjunctive constrainthttps github comhuggingfacetransformersissues14081 where one can allow different forms of each word renormalizelogits bool optional defaults to false whether to renormalize the logits after applying all the logits processors or warpers including the custom ones it s highly recommended to set this flag to true as the search algorithms suppose the score logits are normalized but some logit processors or warpers break the normalization constraints listconstraint optional custom constraints that can be added to the generation to ensure that the output will contain the use of certain tokens as defined by constraint objects in the most sensible way possible forcedbostokenid int optional defaults to model config forcedbostokenid the id of the token to force as the first generated token after the decoderstarttokenid useful for multilingual models like mbart modeldocmbart where the first generated token needs to be the target language token forcedeostokenid unionint listint optional defaults to model config forcedeostokenid the id of the token to force as the last generated token when maxlength is reached optionally use a list to set multiple endofsequence tokens removeinvalidvalues bool optional defaults to model config removeinvalidvalues whether to remove possible nan and inf outputs of the model to prevent the generation method to crash note that using removeinvalidvalues can slow down generation exponentialdecaylengthpenalty tupleint float optional this tuple adds an exponentially increasing length penalty after a certain amount of tokens have been generated the tuple shall consist of startindex decayfactor where startindex indicates where penalty starts and decayfactor represents the factor of exponential decay suppresstokens listint optional a list of tokens that will be suppressed at generation the supresstokens logit processor will set their log probs to inf so that they are not sampled beginsuppresstokens listint optional a list of tokens that will be suppressed at the beginning of the generation the supressbegintokens logit processor will set their log probs to inf so that they are not sampled forceddecoderids listlistint optional a list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling for example 1 123 means the second generated token will always be a token of index 123 sequencebias dicttupleint float optional dictionary that maps a sequence of tokens to its bias term positive biases increase the odds of the sequence being selected while negative biases do the opposite check generation sequencebiaslogitsprocessor for further documentation and examples guidancescale float optional the guidance scale for classifier free guidance cfg cfg is enabled by setting guidancescale 1 higher guidance scale encourages the model to generate samples that are more closely linked to the input prompt usually at the expense of poorer quality lowmemory bool optional switch to sequential topk for contrastive search to reduce peak memory used with contrastive search parameters that define the output variables of generate numreturnsequencesint optional defaults to 1 the number of independently computed returned sequences for each element in the batch outputattentions bool optional defaults to false whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more details outputhiddenstates bool optional defaults to false whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more details outputscores bool optional defaults to false whether or not to return the prediction scores see scores under returned tensors for more details returndictingenerate bool optional defaults to false whether or not to return a utils modeloutput instead of a plain tuple special tokens that can be used at generation time padtokenid int optional the id of the padding token bostokenid int optional the id of the beginningofsequence token eostokenid unionint listint optional the id of the endofsequence token optionally use a list to set multiple endofsequence tokens generation parameters exclusive to encoderdecoder models encodernorepeatngramsize int optional defaults to 0 if set to int 0 all ngrams of that size that occur in the encoderinputids cannot occur in the decoderinputids decoderstarttokenid int optional if an encoderdecoder model starts decoding with a different token than bos the id of that token generation parameters exclusive to assistant generationhttps arxiv orgabs2211 17192 numassistanttokens int optional defaults to 5 defines the number of speculative tokens that shall be generated by the assistant model before being checked by the target model at each iteration higher values for numassistanttokens make the generation more speculative if the assistant model is performant larger speedups can be reached if the assistant model requires lots of corrections lower speedups are reached numassistanttokensschedule str optional defaults to heuristic defines the schedule at which max assistant tokens shall be changed during inference heuristic when all speculative tokens are correct increase numassistanttokens by 2 else reduce by 1 constant numassistanttokens stays unchanged during generation wild card generationkwargs additional generation kwargs will be forwarded to the generate function of the model kwargs that are not present in generate s signature will be used in the model forward pass parameters that control the length of the output if the default maxlength is updated here make sure to update the generate tests following https github comhuggingfacetransformerspull25030 parameters that control the generation strategy used parameters for manipulation of the model output logits parameters that define the output variables of generate special tokens that can be used at generation time generation parameters exclusive to encoderdecoder models assistant generation wild card the remaining attributes do not parametrize generate but are informative andor used by the hub interface additional attributes without default values we don t want to copy values from the model config if we re initializing a generationconfig from a model s default configuration file validate the values of the attributes validates the values of the attributes of the generationconfig instance raises exceptions in the presence of parameterization that can be detected as incorrect from the configuration instance alone note that some parameters are best validated at generate runtime as they may depend on other inputs andor the model such as parameters related to the generation length validation of individual attributes validation of attribute relations 1 detect samplingonly parameterization when not in sampling mode 2 detect beamonly parameterization when not in beam mode 3 detect incorrect paramaterization specific to advanced beam modes constrained beam search group beam search 4 check numreturnsequences 5 check common issue passing generate arguments inside the generation config at save time validate the instance if any warningexception is thrown we refuse to save the instance try with warnings catchwarningsrecordtrue as caughtwarnings self validate for w in caughtwarnings raise valueerrorw message except valueerror as exc warnings warn the generation config instance is invalid validate throws warnings andor exceptions fix these issues to save the configuration this warning will be raised to an exception in v4 34 nnthrown during validation n strexc userwarning return useauthtoken kwargs popuseauthtoken none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if kwargs gettoken none is not none raise valueerror token and useauthtoken are both specified please set only the argument token kwargstoken useauthtoken configfilename configfilename if configfilename is not none else generationconfigname if os path isfilesavedirectory raise assertionerrorfprovided path savedirectory should be a directory not a file os makedirssavedirectory existoktrue if pushtohub commitmessage kwargs popcommitmessage none repoid kwargs poprepoid savedirectory splitos path sep1 repoid self createreporepoid kwargs filestimestamps self getfilestimestampssavedirectory outputconfigfile os path joinsavedirectory configfilename self tojsonfileoutputconfigfile usedifftrue logger infofconfiguration saved in outputconfigfile if pushtohub self uploadmodifiedfiles savedirectory repoid filestimestamps commitmessagecommitmessage tokenkwargs gettoken classmethod def frompretrained cls pretrainedmodelname unionstr os pathlike configfilename optionalunionstr os pathlike none cachedir optionalunionstr os pathlike none forcedownload bool false localfilesonly bool false token optionalunionstr bool none revision str main kwargs generationconfig r instantiate a generationconfig from a generation configuration file args pretrainedmodelname str or os pathlike this can be either a string the model id of a pretrained model configuration hosted inside a model repo on huggingface co valid model ids can be located at the rootlevel like bertbaseuncased or namespaced under a user or organization name like dbmdzbertbasegermancased a path to a directory containing a configuration file saved using the generationconfig savepretrained method e g mymodeldirectory configfilename str or os pathlike optional defaults to generationconfig json name of the generation configuration json file to be loaded from pretrainedmodelname cachedir str or os pathlike optional path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used forcedownload bool optional defaults to false whether or not to force to redownload the configuration files and override the cached versions if they exist resumedownload bool optional defaults to false whether or not to delete incompletely received file attempts to resume the download if such a file exists proxies dictstr str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request token str or bool optional the token to use as http bearer ization for remote files if true or not specified will use the token generated when running huggingfacecli login stored in huggingface revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a gitbased system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revisionrefsprprnumber tip returnunusedkwargs bool optional defaults to false if false then this function returns just the final configuration object if true then this functions returns a tupleconfig unusedkwargs where unusedkwargs is a dictionary consisting of the keyvalue pairs whose keys are not configuration attributes i e the part of kwargs which has not been used to update config and is otherwise ignored subfolder str optional defaults to in case the relevant files are located inside a subfolder of the model repo on huggingface co you can specify the folder name here kwargs dictstr any optional the values in kwargs of any keys which are configuration attributes will be used to override the loaded values behavior concerning keyvalue pairs whose keys are not configuration attributes is controlled by the returnunusedkwargs keyword parameter returns generationconfig the configuration object instantiated from this pretrained model examples python from transformers import generationconfig download configuration from huggingface co and cache generationconfig generationconfig frompretrainedgpt2 e g config was saved using savepretrained testsavedmodel generationconfig savepretrained testsavedmodel generationconfig generationconfig frompretrained testsavedmodel you can also specify configuration names to your generation configuration file generationconfig savepretrained testsavedmodel configfilenamemyconfiguration json generationconfig generationconfig frompretrained testsavedmodel myconfiguration json if you d like to try a minor variation to an existing configuration you can also pass generation arguments to frompretrained be mindful that typos and unused arguments will be ignored generationconfig unusedkwargs generationconfig frompretrained gpt2 topk1 foofalse dosampletrue returnunusedkwargstrue generationconfig topk 1 unusedkwargs foo false configfilename configfilename if configfilename is not none else generationconfigname resumedownload kwargs popresumedownload false proxies kwargs popproxies none useauthtoken kwargs popuseauthtoken none subfolder kwargs popsubfolder frompipeline kwargs popfrompipeline none fromautoclass kwargs popfromauto false commithash kwargs popcommithash none if useauthtoken is not none warnings warn the useauthtoken argument is deprecated and will be removed in v5 of transformers please use token instead futurewarning if token is not none raise valueerror token and useauthtoken are both specified please set only the argument token token useauthtoken useragent filetype config fromautoclass fromautoclass if frompipeline is not none useragentusingpipeline frompipeline configpath os path joinpretrainedmodelname configfilename configpath strconfigpath islocal os path existsconfigpath if os path isfileos path joinsubfolder configpath special case when configpath is a local file resolvedconfigfile configpath islocal true elif isremoteurlconfigpath configurationfile configpath resolvedconfigfile downloadurlconfigpath else configurationfile configfilename try load from local folder or from cache or download from model hub and cache resolvedconfigfile cachedfile pretrainedmodelname configurationfile cachedircachedir forcedownloadforcedownload proxiesproxies resumedownloadresumedownload localfilesonlylocalfilesonly tokentoken useragentuseragent revisionrevision subfoldersubfolder commithashcommithash commithash extractcommithashresolvedconfigfile commithash except environmenterror raise any environment error raise by cachedfile it will have a helpful error message adapted to the original exception raise except exception for any other exception we throw a generic error raise environmenterror fcan t load the configuration of pretrainedmodelname if you were trying to load it from https huggingface comodels make sure you don t have a local directory with the same f name otherwise make sure pretrainedmodelname is the correct path to a directory f containing a configurationfile file try load config dict configdict cls dictfromjsonfileresolvedconfigfile configdictcommithash commithash except json jsondecodeerror unicodedecodeerror raise environmenterror fit looks like the config file at resolvedconfigfile is not a valid json file if islocal logger infofloading configuration file resolvedconfigfile else logger infofloading configuration file configurationfile from cache at resolvedconfigfile if kwargs getreturnunusedkwargs is true config unusedkwargs cls fromdictconfigdict kwargs config originalobjecthash hashconfig hash to detect whether the instance was modified return config unusedkwargs else config cls fromdictconfigdict kwargs config originalobjecthash hashconfig hash to detect whether the instance was modified return config classmethod def dictfromjsonfilecls jsonfile unionstr os pathlike with openjsonfile r encodingutf8 as reader text reader read return json loadstext classmethod def fromdictcls configdict dictstr any kwargs generationconfig returnunusedkwargs kwargs popreturnunusedkwargs false those arguments may be passed along for our internal telemetry we remove them so they don t appear in returnunusedkwargs kwargs popfromauto none kwargs popfrompipeline none the commit hash might have been updated in the configdict we don t want the kwargs to erase that update if commithash in kwargs and commithash in configdict kwargscommithash configdictcommithash the line below allows modelspecific config to be loaded as well through kwargs with safety checks see https github comhuggingfacetransformerspull21269 config clsconfigdict kwargs unusedkwargs config updatekwargs logger infofgenerate config config if returnunusedkwargs return config unusedkwargs else return config def dicttorchdtypetostrself d dictstr any none if d gettorchdtype none is not none and not isinstancedtorchdtype str dtorchdtype strdtorchdtype split 1 for value in d values if isinstancevalue dict self dicttorchdtypetostrvalue def todiffdictself dictstr any configdict self todict get the default config dict defaultconfigdict generationconfig todict serializableconfigdict only serialize values that differ from the default config for key value in configdict items if key not in defaultconfigdict or key transformersversion or value defaultconfigdictkey serializableconfigdictkey value self dicttorchdtypetostrserializableconfigdict return serializableconfigdict def todictself dictstr any output copy deepcopyself dict fields to ignore at serialization time if commithash in output del outputcommithash if originalobjecthash in output del outputoriginalobjecthash transformers version when serializing this file outputtransformersversion version self dicttorchdtypetostroutput return output def tojsonstringself usediff bool true ignoremetadata bool false str if usediff is true configdict self todiffdict else configdict self todict if ignoremetadata for metadatafield in metadatafields configdict popmetadatafield none return json dumpsconfigdict indent2 sortkeystrue n def tojsonfileself jsonfilepath unionstr os pathlike usediff bool true with openjsonfilepath w encodingutf8 as writer writer writeself tojsonstringusediffusediff classmethod def frommodelconfigcls modelconfig pretrainedconfig generationconfig configdict modelconfig todict configdict popfrommodelconfig none config cls fromdictconfigdict returnunusedkwargsfalse frommodelconfigtrue special case some models have generation attributes set in the decoder use them if still unset in the generation config for decodername in decoder generator textconfig if decodername in configdict defaultgenerationconfig generationconfig decoderconfig configdictdecodername for attr in config todict keys if attr in decoderconfig and getattrconfig attr getattrdefaultgenerationconfig attr setattrconfig attr decoderconfigattr config originalobjecthash hashconfig hash to detect whether the instance was modified return config def updateself kwargs toremove for key value in kwargs items if hasattrself key setattrself key value toremove appendkey remove all the attributes that were updated without modifying the input dict unusedkwargs key value for key value in kwargs items if key not in toremove return unusedkwargs coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license generation configuration class and utilities no format class that holds a configuration for a generation task a generate call supports the following generation methods for text decoder text to text speech to text and vision to text models greedy decoding by calling generation generationmixin greedy_search if num_beams 1 and do_sample false contrastive search by calling generation generationmixin contrastive_search if penalty_alpha 0 and top_k 1 multinomial sampling by calling generation generationmixin sample if num_beams 1 and do_sample true beam search decoding by calling generation generationmixin beam_search if num_beams 1 and do_sample false beam search multinomial sampling by calling generation generationmixin beam_sample if num_beams 1 and do_sample true diverse beam search decoding by calling generation generationmixin group_beam_search if num_beams 1 and num_beam_groups 1 constrained beam search decoding by calling generation generationmixin constrained_beam_search if constraints none or force_words_ids none assisted decoding by calling generation generationmixin assisted_decoding if assistant_model is passed to generate you do not need to call any of the above methods directly pass custom parameter values to generate to learn more about decoding strategies refer to the text generation strategies guide generation_strategies arg parameters that control the length of the output max_length int optional defaults to 20 the maximum length the generated tokens can have corresponds to the length of the input prompt max_new_tokens its effect is overridden by max_new_tokens if also set max_new_tokens int optional the maximum numbers of tokens to generate ignoring the number of tokens in the prompt min_length int optional defaults to 0 the minimum length of the sequence to be generated corresponds to the length of the input prompt min_new_tokens its effect is overridden by min_new_tokens if also set min_new_tokens int optional the minimum numbers of tokens to generate ignoring the number of tokens in the prompt early_stopping bool or str optional defaults to false controls the stopping condition for beam based methods like beam search it accepts the following values true where the generation stops as soon as there are num_beams complete candidates false where an heuristic is applied and the generation stops when is it very unlikely to find better candidates never where the beam search procedure only stops when there cannot be better candidates canonical beam search algorithm max_time float optional the maximum amount of time you allow the computation to run for in seconds generation will still finish the current pass after allocated time has been passed parameters that control the generation strategy used do_sample bool optional defaults to false whether or not to use sampling use greedy decoding otherwise num_beams int optional defaults to 1 number of beams for beam search 1 means no beam search num_beam_groups int optional defaults to 1 number of groups to divide num_beams into in order to ensure diversity among different groups of beams this paper https arxiv org pdf 1610 02424 pdf for more details penalty_alpha float optional the values balance the model confidence and the degeneration penalty in contrastive search decoding use_cache bool optional defaults to true whether or not the model should use the past last key values attentions if applicable to the model to speed up decoding parameters for manipulation of the model output logits temperature float optional defaults to 1 0 the value used to modulate the next token probabilities top_k int optional defaults to 50 the number of highest probability vocabulary tokens to keep for top k filtering top_p float optional defaults to 1 0 if set to float 1 only the smallest set of most probable tokens with probabilities that add up to top_p or higher are kept for generation typical_p float optional defaults to 1 0 local typicality measures how similar the conditional probability of predicting a target token next is to the expected conditional probability of predicting a random token next given the partial text already generated if set to float 1 the smallest set of the most locally typical tokens with probabilities that add up to typical_p or higher are kept for generation see this paper https arxiv org pdf 2202 00666 pdf for more details epsilon_cutoff float optional defaults to 0 0 if set to float strictly between 0 and 1 only tokens with a conditional probability greater than epsilon_cutoff will be sampled in the paper suggested values range from 3e 4 to 9e 4 depending on the size of the model see truncation sampling as language model desmoothing https arxiv org abs 2210 15191 for more details eta_cutoff float optional defaults to 0 0 eta sampling is a hybrid of locally typical sampling and epsilon sampling if set to float strictly between 0 and 1 a token is only considered if it is greater than either eta_cutoff or sqrt eta_cutoff exp entropy softmax next_token_logits the latter term is intuitively the expected next token probability scaled by sqrt eta_cutoff in the paper suggested values range from 3e 4 to 2e 3 depending on the size of the model see truncation sampling as language model desmoothing https arxiv org abs 2210 15191 for more details diversity_penalty float optional defaults to 0 0 this value is subtracted from a beam s score if it generates a token same as any beam from other group at a particular time note that diversity_penalty is only effective if group beam search is enabled repetition_penalty float optional defaults to 1 0 the parameter for repetition penalty 1 0 means no penalty see this paper https arxiv org pdf 1909 05858 pdf for more details encoder_repetition_penalty float optional defaults to 1 0 the paramater for encoder_repetition_penalty an exponential penalty on sequences that are not in the original input 1 0 means no penalty length_penalty float optional defaults to 1 0 exponential penalty to the length that is used with beam based generation it is applied as an exponent to the sequence length which in turn is used to divide the score of the sequence since the score is the log likelihood of the sequence i e negative length_penalty 0 0 promotes longer sequences while length_penalty 0 0 encourages shorter sequences no_repeat_ngram_size int optional defaults to 0 if set to int 0 all ngrams of that size can only occur once bad_words_ids list list int optional list of list of token ids that are not allowed to be generated check generation nobadwordslogitsprocessor for further documentation and examples force_words_ids list list int or list list list int optional list of token ids that must be generated if given a list list int this is treated as a simple list of words that must be included the opposite to bad_words_ids if given list list list int this triggers a disjunctive constraint https github com huggingface transformers issues 14081 where one can allow different forms of each word renormalize_logits bool optional defaults to false whether to renormalize the logits after applying all the logits processors or warpers including the custom ones it s highly recommended to set this flag to true as the search algorithms suppose the score logits are normalized but some logit processors or warpers break the normalization constraints list constraint optional custom constraints that can be added to the generation to ensure that the output will contain the use of certain tokens as defined by constraint objects in the most sensible way possible forced_bos_token_id int optional defaults to model config forced_bos_token_id the id of the token to force as the first generated token after the decoder_start_token_id useful for multilingual models like mbart model_doc mbart where the first generated token needs to be the target language token forced_eos_token_id union int list int optional defaults to model config forced_eos_token_id the id of the token to force as the last generated token when max_length is reached optionally use a list to set multiple end of sequence tokens remove_invalid_values bool optional defaults to model config remove_invalid_values whether to remove possible nan and inf outputs of the model to prevent the generation method to crash note that using remove_invalid_values can slow down generation exponential_decay_length_penalty tuple int float optional this tuple adds an exponentially increasing length penalty after a certain amount of tokens have been generated the tuple shall consist of start_index decay_factor where start_index indicates where penalty starts and decay_factor represents the factor of exponential decay suppress_tokens list int optional a list of tokens that will be suppressed at generation the supresstokens logit processor will set their log probs to inf so that they are not sampled begin_suppress_tokens list int optional a list of tokens that will be suppressed at the beginning of the generation the supressbegintokens logit processor will set their log probs to inf so that they are not sampled forced_decoder_ids list list int optional a list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling for example 1 123 means the second generated token will always be a token of index 123 sequence_bias dict tuple int float optional dictionary that maps a sequence of tokens to its bias term positive biases increase the odds of the sequence being selected while negative biases do the opposite check generation sequencebiaslogitsprocessor for further documentation and examples guidance_scale float optional the guidance scale for classifier free guidance cfg cfg is enabled by setting guidance_scale 1 higher guidance scale encourages the model to generate samples that are more closely linked to the input prompt usually at the expense of poorer quality low_memory bool optional switch to sequential topk for contrastive search to reduce peak memory used with contrastive search parameters that define the output variables of generate num_return_sequences int optional defaults to 1 the number of independently computed returned sequences for each element in the batch output_attentions bool optional defaults to false whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more details output_hidden_states bool optional defaults to false whether or not to return the hidden states of all layers see hidden_states under returned tensors for more details output_scores bool optional defaults to false whether or not to return the prediction scores see scores under returned tensors for more details return_dict_in_generate bool optional defaults to false whether or not to return a utils modeloutput instead of a plain tuple special tokens that can be used at generation time pad_token_id int optional the id of the padding token bos_token_id int optional the id of the beginning of sequence token eos_token_id union int list int optional the id of the end of sequence token optionally use a list to set multiple end of sequence tokens generation parameters exclusive to encoder decoder models encoder_no_repeat_ngram_size int optional defaults to 0 if set to int 0 all ngrams of that size that occur in the encoder_input_ids cannot occur in the decoder_input_ids decoder_start_token_id int optional if an encoder decoder model starts decoding with a different token than bos the id of that token generation parameters exclusive to assistant generation https arxiv org abs 2211 17192 num_assistant_tokens int optional defaults to 5 defines the number of _speculative tokens_ that shall be generated by the assistant model before being checked by the target model at each iteration higher values for num_assistant_tokens make the generation more _speculative_ if the assistant model is performant larger speed ups can be reached if the assistant model requires lots of corrections lower speed ups are reached num_assistant_tokens_schedule str optional defaults to heuristic defines the schedule at which max assistant tokens shall be changed during inference _heuristic_ when all _speculative_ tokens are correct increase num_assistant_tokens by 2 else reduce by 1 constant num_assistant_tokens stays unchanged during generation wild card generation_kwargs additional generation kwargs will be forwarded to the generate function of the model kwargs that are not present in generate s signature will be used in the model forward pass parameters that control the length of the output if the default max_length is updated here make sure to update the generate tests following https github com huggingface transformers pull 25030 parameters that control the generation strategy used parameters for manipulation of the model output logits parameters that define the output variables of generate special tokens that can be used at generation time generation parameters exclusive to encoder decoder models assistant generation wild card the remaining attributes do not parametrize generate but are informative and or used by the hub interface additional attributes without default values we don t want to copy values from the model config if we re initializing a generationconfig from a model s default configuration file validate the values of the attributes validates the values of the attributes of the generationconfig instance raises exceptions in the presence of parameterization that can be detected as incorrect from the configuration instance alone note that some parameters are best validated at generate runtime as they may depend on other inputs and or the model such as parameters related to the generation length validation of individual attributes validation of attribute relations 1 detect sampling only parameterization when not in sampling mode contrastive search uses top_k 2 detect beam only parameterization when not in beam mode 3 detect incorrect paramaterization specific to advanced beam modes constrained beam search group beam search 4 check num_return_sequences 5 check common issue passing generate arguments inside the generation config save a generation configuration object to the directory save_directory so that it can be re loaded using the generationconfig from_pretrained class method args save_directory str or os pathlike directory where the configuration json file will be saved will be created if it does not exist config_file_name str or os pathlike optional defaults to generation_config json name of the generation configuration json file to be saved in save_directory push_to_hub bool optional defaults to false whether or not to push your model to the hugging face model hub after saving it you can specify the repository you want to push to with repo_id will default to the name of save_directory in your namespace kwargs dict str any optional additional key word arguments passed along to the utils pushtohubmixin push_to_hub method at save time validate the instance if any warning exception is thrown we refuse to save the instance instantiate a generationconfig from a generation configuration file args pretrained_model_name str or os pathlike this can be either a string the model id of a pretrained model configuration hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing a configuration file saved using the generationconfig save_pretrained method e g my_model_directory config_file_name str or os pathlike optional defaults to generation_config json name of the generation configuration json file to be loaded from pretrained_model_name cache_dir str or os pathlike optional path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used force_download bool optional defaults to false whether or not to force to re download the configuration files and override the cached versions if they exist resume_download bool optional defaults to false whether or not to delete incompletely received file attempts to resume the download if such a file exists proxies dict str str optional a dictionary of proxy servers to use by protocol or endpoint e g http foo bar 3128 http hostname foo bar 4012 the proxies are used on each request token str or bool optional the token to use as http bearer ization for remote files if true or not specified will use the token generated when running huggingface cli login stored in huggingface revision str optional defaults to main the specific model version to use it can be a branch name a tag name or a commit id since we use a git based system for storing models and other artifacts on huggingface co so revision can be any identifier allowed by git tip to test a pull request you made on the hub you can pass revision refs pr pr_number tip return_unused_kwargs bool optional defaults to false if false then this function returns just the final configuration object if true then this functions returns a tuple config unused_kwargs where unused_kwargs is a dictionary consisting of the key value pairs whose keys are not configuration attributes i e the part of kwargs which has not been used to update config and is otherwise ignored subfolder str optional defaults to in case the relevant files are located inside a subfolder of the model repo on huggingface co you can specify the folder name here kwargs dict str any optional the values in kwargs of any keys which are configuration attributes will be used to override the loaded values behavior concerning key value pairs whose keys are not configuration attributes is controlled by the return_unused_kwargs keyword parameter returns generationconfig the configuration object instantiated from this pretrained model examples python from transformers import generationconfig download configuration from huggingface co and cache generation_config generationconfig from_pretrained gpt2 e g config was saved using save_pretrained test saved_model generation_config save_pretrained test saved_model generation_config generationconfig from_pretrained test saved_model you can also specify configuration names to your generation configuration file generation_config save_pretrained test saved_model config_file_name my_configuration json generation_config generationconfig from_pretrained test saved_model my_configuration json if you d like to try a minor variation to an existing configuration you can also pass generation arguments to from_pretrained be mindful that typos and unused arguments will be ignored generation_config unused_kwargs generationconfig from_pretrained gpt2 top_k 1 foo false do_sample true return_unused_kwargs true generation_config top_k 1 unused_kwargs foo false special case when config_path is a local file load from local folder or from cache or download from model hub and cache raise any environment error raise by cached_file it will have a helpful error message adapted to the original exception for any other exception we throw a generic error load config dict hash to detect whether the instance was modified hash to detect whether the instance was modified instantiates a generationconfig from a python dictionary of parameters args config_dict dict str any dictionary that will be used to instantiate the configuration object kwargs dict str any additional parameters from which to initialize the configuration object returns generationconfig the configuration object instantiated from those parameters those arguments may be passed along for our internal telemetry we remove them so they don t appear in return_unused_kwargs the commit hash might have been updated in the config_dict we don t want the kwargs to erase that update the line below allows model specific config to be loaded as well through kwargs with safety checks see https github com huggingface transformers pull 21269 checks whether the passed dictionary and its nested dicts have a torch_dtype key and if it s not none converts torch dtype to a string of just the type for example torch float32 get converted into float32 string which can then be stored in the json format removes all attributes from config which correspond to the default config attributes for better readability and serializes to a python dictionary returns dict str any dictionary of all the attributes that make up this configuration instance get the default config dict only serialize values that differ from the default config serializes this instance to a python dictionary returns dict str any dictionary of all the attributes that make up this configuration instance fields to ignore at serialization time transformers version when serializing this file serializes this instance to a json string args use_diff bool optional defaults to true if set to true only the difference between the config instance and the default generationconfig is serialized to json string ignore_metadata bool optional defaults to false whether to ignore the metadata fields present in the instance returns str string containing all the attributes that make up this configuration instance in json format save this instance to a json file args json_file_path str or os pathlike path to the json file in which this configuration instance s parameters will be saved use_diff bool optional defaults to true if set to true only the difference between the config instance and the default generationconfig is serialized to json file instantiates a generationconfig from a pretrainedconfig this function is useful to convert legacy pretrainedconfig objects which may contain generation parameters into a stand alone generationconfig args model_config pretrainedconfig the model config that will be used to instantiate the generation config returns generationconfig the configuration object instantiated from those parameters special case some models have generation attributes set in the decoder use them if still unset in the generation config hash to detect whether the instance was modified updates attributes of this class instance with attributes from kwargs if they match existing atributtes returning all the unused kwargs args kwargs dict str any dictionary of attributes to tentatively update this class returns dict str any dictionary containing all the key value pairs that were not used to update the instance remove all the attributes that were updated without modifying the input dict
import copy
import json
import os
import warnings
from typing import Any, Dict, Optional, Union

from .. import __version__
from ..configuration_utils import PretrainedConfig
from ..utils import (
    GENERATION_CONFIG_NAME,
    PushToHubMixin,
    cached_file,
    download_url,
    extract_commit_hash,
    is_remote_url,
    logging,
)


logger = logging.get_logger(__name__)
METADATA_FIELDS = ("_from_model_config", "_commit_hash", "_original_object_hash", "transformers_version")


class GenerationConfig(PushToHubMixin):
    
    r

    def __init__(self, **kwargs):
        
        
        self.max_length = kwargs.pop("max_length", 20)
        self.max_new_tokens = kwargs.pop("max_new_tokens", None)
        self.min_length = kwargs.pop("min_length", 0)
        self.min_new_tokens = kwargs.pop("min_new_tokens", None)
        self.early_stopping = kwargs.pop("early_stopping", False)
        self.max_time = kwargs.pop("max_time", None)

        
        self.do_sample = kwargs.pop("do_sample", False)
        self.num_beams = kwargs.pop("num_beams", 1)
        self.num_beam_groups = kwargs.pop("num_beam_groups", 1)
        self.penalty_alpha = kwargs.pop("penalty_alpha", None)
        self.use_cache = kwargs.pop("use_cache", True)

        
        self.temperature = kwargs.pop("temperature", 1.0)
        self.top_k = kwargs.pop("top_k", 50)
        self.top_p = kwargs.pop("top_p", 1.0)
        self.typical_p = kwargs.pop("typical_p", 1.0)
        self.epsilon_cutoff = kwargs.pop("epsilon_cutoff", 0.0)
        self.eta_cutoff = kwargs.pop("eta_cutoff", 0.0)
        self.diversity_penalty = kwargs.pop("diversity_penalty", 0.0)
        self.repetition_penalty = kwargs.pop("repetition_penalty", 1.0)
        self.encoder_repetition_penalty = kwargs.pop("encoder_repetition_penalty", 1.0)
        self.length_penalty = kwargs.pop("length_penalty", 1.0)
        self.no_repeat_ngram_size = kwargs.pop("no_repeat_ngram_size", 0)
        self.bad_words_ids = kwargs.pop("bad_words_ids", None)
        self.force_words_ids = kwargs.pop("force_words_ids", None)
        self.renormalize_logits = kwargs.pop("renormalize_logits", False)
        self.constraints = kwargs.pop("constraints", None)
        self.forced_bos_token_id = kwargs.pop("forced_bos_token_id", None)
        self.forced_eos_token_id = kwargs.pop("forced_eos_token_id", None)
        self.remove_invalid_values = kwargs.pop("remove_invalid_values", False)
        self.exponential_decay_length_penalty = kwargs.pop("exponential_decay_length_penalty", None)
        self.suppress_tokens = kwargs.pop("suppress_tokens", None)
        self.begin_suppress_tokens = kwargs.pop("begin_suppress_tokens", None)
        self.forced_decoder_ids = kwargs.pop("forced_decoder_ids", None)
        self.sequence_bias = kwargs.pop("sequence_bias", None)
        self.guidance_scale = kwargs.pop("guidance_scale", None)
        self.low_memory = kwargs.pop("low_memory", None)

        
        self.num_return_sequences = kwargs.pop("num_return_sequences", 1)
        self.output_attentions = kwargs.pop("output_attentions", False)
        self.output_hidden_states = kwargs.pop("output_hidden_states", False)
        self.output_scores = kwargs.pop("output_scores", False)
        self.return_dict_in_generate = kwargs.pop("return_dict_in_generate", False)

        
        self.pad_token_id = kwargs.pop("pad_token_id", None)
        self.bos_token_id = kwargs.pop("bos_token_id", None)
        self.eos_token_id = kwargs.pop("eos_token_id", None)

        
        self.encoder_no_repeat_ngram_size = kwargs.pop("encoder_no_repeat_ngram_size", 0)
        self.decoder_start_token_id = kwargs.pop("decoder_start_token_id", None)

        
        self.num_assistant_tokens = kwargs.pop("num_assistant_tokens", 5)
        self.num_assistant_tokens_schedule = kwargs.pop("num_assistant_tokens_schedule", "heuristic")

        
        self.generation_kwargs = kwargs.pop("generation_kwargs", {})

        
        
        self._from_model_config = kwargs.pop("_from_model_config", False)
        self._commit_hash = kwargs.pop("_commit_hash", None)
        self.transformers_version = kwargs.pop("transformers_version", __version__)

        
        if not self._from_model_config:
            
            
            for key, value in kwargs.items():
                try:
                    setattr(self, key, value)
                except AttributeError as err:
                    logger.error(f"Can't set {key} with value {value} for {self}")
                    raise err

        
        self.validate(is_init=True)

    def __hash__(self):
        return hash(self.to_json_string(ignore_metadata=True))

    def __eq__(self, other):
        if not isinstance(other, GenerationConfig):
            return False

        self_without_metadata = self.to_json_string(use_diff=False, ignore_metadata=True)
        other_without_metadata = other.to_json_string(use_diff=False, ignore_metadata=True)
        return self_without_metadata == other_without_metadata

    def __repr__(self):
        return f"{self.__class__.__name__} {self.to_json_string(ignore_metadata=True)}"

    def validate(self, is_init=False):
        

        
        if self.early_stopping not in {True, False, "never"}:
            raise ValueError(f"`early_stopping` must be a boolean or 'never', but is {self.early_stopping}.")

        
        fix_location = ""
        if is_init:
            fix_location = (
                " This was detected when initializing the generation config instance, which means the corresponding "
                "file may hold incorrect parameterization and should be fixed."
            )

        
        if self.do_sample is False:
            greedy_wrong_parameter_msg = (
                "`do_sample` is set to `False`. However, `{flag_name}` is set to `{flag_value}` -- this flag is only "
                "used in sample-based generation modes. You should set `do_sample=True` or unset `{flag_name}`."
                + fix_location
            )
            if self.temperature != 1.0:
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="temperature", flag_value=self.temperature),
                    UserWarning,
                )
            if self.top_p != 1.0:
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="top_p", flag_value=self.top_p),
                    UserWarning,
                )
            if self.typical_p != 1.0:
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="typical_p", flag_value=self.typical_p),
                    UserWarning,
                )
            if self.top_k != 50 and self.penalty_alpha is None:  
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="top_k", flag_value=self.top_k),
                    UserWarning,
                )
            if self.epsilon_cutoff != 0.0:
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="epsilon_cutoff", flag_value=self.epsilon_cutoff),
                    UserWarning,
                )
            if self.eta_cutoff != 0.0:
                warnings.warn(
                    greedy_wrong_parameter_msg.format(flag_name="eta_cutoff", flag_value=self.eta_cutoff),
                    UserWarning,
                )

        
        if self.num_beams is None:
            warnings.warn("`num_beams` is set to None - defaulting to 1.", UserWarning)
            self.num_beams = 1

        if self.num_beams == 1:
            single_beam_wrong_parameter_msg = (
                "`num_beams` is set to 1. However, `{flag_name}` is set to `{flag_value}` -- this flag is only used "
                "in beam-based generation modes. You should set `num_beams>1` or unset `{flag_name}`." + fix_location
            )
            if self.early_stopping is not False:
                warnings.warn(
                    single_beam_wrong_parameter_msg.format(flag_name="early_stopping", flag_value=self.early_stopping),
                    UserWarning,
                )
            if self.num_beam_groups != 1:
                warnings.warn(
                    single_beam_wrong_parameter_msg.format(
                        flag_name="num_beam_groups", flag_value=self.num_beam_groups
                    ),
                    UserWarning,
                )
            if self.diversity_penalty != 0.0:
                warnings.warn(
                    single_beam_wrong_parameter_msg.format(
                        flag_name="diversity_penalty", flag_value=self.diversity_penalty
                    ),
                    UserWarning,
                )
            if self.length_penalty != 1.0:
                warnings.warn(
                    single_beam_wrong_parameter_msg.format(flag_name="length_penalty", flag_value=self.length_penalty),
                    UserWarning,
                )
            if self.constraints is not None:
                warnings.warn(
                    single_beam_wrong_parameter_msg.format(flag_name="constraints", flag_value=self.constraints),
                    UserWarning,
                )

        
        else:
            
            if self.constraints is not None:
                constrained_wrong_parameter_msg = (
                    "`constraints` is not `None`, triggering constrained beam search. However, `{flag_name}` is set "
                    "to `{flag_value}`, which is incompatible with this generation mode. Set `constraints=None` or "
                    "unset `{flag_name}` to continue." + fix_location
                )
                if self.do_sample is True:
                    raise ValueError(
                        constrained_wrong_parameter_msg.format(flag_name="do_sample", flag_value=self.do_sample)
                    )
                if self.num_beam_groups != 1:
                    raise ValueError(
                        constrained_wrong_parameter_msg.format(
                            flag_name="num_beam_groups", flag_value=self.num_beam_groups
                        )
                    )
            
            if self.diversity_penalty != 0.0 or self.num_beam_groups != 1:
                group_error_prefix = (
                    "`diversity_penalty` is not 0.0 or `num_beam_groups` is not 1, triggering group beam search. In "
                    "this generation mode, "
                )
                if self.do_sample is True:
                    raise ValueError(group_error_prefix + "`do_sample` must be set to `False`")
                if self.num_beams % self.num_beam_groups != 0:
                    raise ValueError(group_error_prefix + "`num_beams` should be divisible by `num_beam_groups`")
                if self.diversity_penalty == 0.0:
                    raise ValueError(
                        group_error_prefix
                        + "`diversity_penalty` should be greater than `0.0`, otherwise your groups will be identical."
                    )

        
        if self.num_return_sequences != 1:
            if self.num_beams == 1:
                if self.do_sample is False:
                    raise ValueError(
                        "Greedy methods without beam search do not support `num_return_sequences` different than 1 "
                        f"(got {self.num_return_sequences})."
                    )
            elif self.num_return_sequences > self.num_beams:
                raise ValueError(
                    f"`num_return_sequences` ({self.num_return_sequences}) has to be smaller or equal to `num_beams` "
                    f"({self.num_beams})."
                )

        
        generate_arguments = (
            "logits_processor",
            "stopping_criteria",
            "prefix_allowed_tokens_fn",
            "synced_gpus",
            "assistant_model",
            "streamer",
            "negative_prompt_ids",
            "negative_prompt_attention_mask",
        )
        for arg in generate_arguments:
            if hasattr(self, arg):
                raise ValueError(
                    f"Argument `{arg}` is not a valid argument of `GenerationConfig`. It should be passed to "
                    "`generate()` (or a pipeline) directly."
                )

    def save_pretrained(
        self,
        save_directory: Union[str, os.PathLike],
        config_file_name: Optional[Union[str, os.PathLike]] = None,
        push_to_hub: bool = False,
        **kwargs,
    ):
        r

        
        try:
            with warnings.catch_warnings(record=True) as caught_warnings:
                self.validate()
            for w in caught_warnings:
                raise ValueError(w.message)
        except ValueError as exc:
            warnings.warn(
                "The generation config instance is invalid -- `.validate()` throws warnings and/or exceptions. "
                "Fix these issues to save the configuration. This warning will be raised to an exception in v4.34."
                "\n\nThrown during validation:\n" + str(exc),
                UserWarning,
            )
            return

        use_auth_token = kwargs.pop("use_auth_token", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if kwargs.get("token", None) is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            kwargs["token"] = use_auth_token

        config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME

        if os.path.isfile(save_directory):
            raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")

        os.makedirs(save_directory, exist_ok=True)

        if push_to_hub:
            commit_message = kwargs.pop("commit_message", None)
            repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1])
            repo_id = self._create_repo(repo_id, **kwargs)
            files_timestamps = self._get_files_timestamps(save_directory)

        output_config_file = os.path.join(save_directory, config_file_name)

        self.to_json_file(output_config_file, use_diff=True)
        logger.info(f"Configuration saved in {output_config_file}")

        if push_to_hub:
            self._upload_modified_files(
                save_directory,
                repo_id,
                files_timestamps,
                commit_message=commit_message,
                token=kwargs.get("token"),
            )

    @classmethod
    def from_pretrained(
        cls,
        pretrained_model_name: Union[str, os.PathLike],
        config_file_name: Optional[Union[str, os.PathLike]] = None,
        cache_dir: Optional[Union[str, os.PathLike]] = None,
        force_download: bool = False,
        local_files_only: bool = False,
        token: Optional[Union[str, bool]] = None,
        revision: str = "main",
        **kwargs,
    ) -> "GenerationConfig":
        r
        config_file_name = config_file_name if config_file_name is not None else GENERATION_CONFIG_NAME

        resume_download = kwargs.pop("resume_download", False)
        proxies = kwargs.pop("proxies", None)
        use_auth_token = kwargs.pop("use_auth_token", None)
        subfolder = kwargs.pop("subfolder", "")
        from_pipeline = kwargs.pop("_from_pipeline", None)
        from_auto_class = kwargs.pop("_from_auto", False)
        commit_hash = kwargs.pop("_commit_hash", None)

        if use_auth_token is not None:
            warnings.warn(
                "The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.",
                FutureWarning,
            )
            if token is not None:
                raise ValueError(
                    "`token` and `use_auth_token` are both specified. Please set only the argument `token`."
                )
            token = use_auth_token

        user_agent = {"file_type": "config", "from_auto_class": from_auto_class}
        if from_pipeline is not None:
            user_agent["using_pipeline"] = from_pipeline

        config_path = os.path.join(pretrained_model_name, config_file_name)
        config_path = str(config_path)

        is_local = os.path.exists(config_path)
        if os.path.isfile(os.path.join(subfolder, config_path)):
            
            resolved_config_file = config_path
            is_local = True
        elif is_remote_url(config_path):
            configuration_file = config_path
            resolved_config_file = download_url(config_path)
        else:
            configuration_file = config_file_name
            try:
                
                resolved_config_file = cached_file(
                    pretrained_model_name,
                    configuration_file,
                    cache_dir=cache_dir,
                    force_download=force_download,
                    proxies=proxies,
                    resume_download=resume_download,
                    local_files_only=local_files_only,
                    token=token,
                    user_agent=user_agent,
                    revision=revision,
                    subfolder=subfolder,
                    _commit_hash=commit_hash,
                )
                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
            except EnvironmentError:
                
                
                raise
            except Exception:
                
                raise EnvironmentError(
                    f"Can't load the configuration of '{pretrained_model_name}'. If you were trying to load it"
                    " from 'https://huggingface.co/models', make sure you don't have a local directory with the same"
                    f" name. Otherwise, make sure '{pretrained_model_name}' is the correct path to a directory"
                    f" containing a {configuration_file} file"
                )

        try:
            
            config_dict = cls._dict_from_json_file(resolved_config_file)
            config_dict["_commit_hash"] = commit_hash
        except (json.JSONDecodeError, UnicodeDecodeError):
            raise EnvironmentError(
                f"It looks like the config file at '{resolved_config_file}' is not a valid JSON file."
            )

        if is_local:
            logger.info(f"loading configuration file {resolved_config_file}")
        else:
            logger.info(f"loading configuration file {configuration_file} from cache at {resolved_config_file}")

        if kwargs.get("return_unused_kwargs") is True:
            config, unused_kwargs = cls.from_dict(config_dict, **kwargs)
            config._original_object_hash = hash(config)  
            return config, unused_kwargs
        else:
            config = cls.from_dict(config_dict, **kwargs)
            config._original_object_hash = hash(config)  
            return config

    @classmethod
    def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
        with open(json_file, "r", encoding="utf-8") as reader:
            text = reader.read()
        return json.loads(text)

    @classmethod
    def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "GenerationConfig":
        
        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
        
        
        kwargs.pop("_from_auto", None)
        kwargs.pop("_from_pipeline", None)
        
        if "_commit_hash" in kwargs and "_commit_hash" in config_dict:
            kwargs["_commit_hash"] = config_dict["_commit_hash"]

        
        
        config = cls(**{**config_dict, **kwargs})
        unused_kwargs = config.update(**kwargs)

        logger.info(f"Generate config {config}")
        if return_unused_kwargs:
            return config, unused_kwargs
        else:
            return config

    def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None:
        
        if d.get("torch_dtype", None) is not None and not isinstance(d["torch_dtype"], str):
            d["torch_dtype"] = str(d["torch_dtype"]).split(".")[1]
        for value in d.values():
            if isinstance(value, dict):
                self.dict_torch_dtype_to_str(value)

    def to_diff_dict(self) -> Dict[str, Any]:
        
        config_dict = self.to_dict()

        
        default_config_dict = GenerationConfig().to_dict()

        serializable_config_dict = {}

        
        for key, value in config_dict.items():
            if key not in default_config_dict or key == "transformers_version" or value != default_config_dict[key]:
                serializable_config_dict[key] = value

        self.dict_torch_dtype_to_str(serializable_config_dict)
        return serializable_config_dict

    def to_dict(self) -> Dict[str, Any]:
        
        output = copy.deepcopy(self.__dict__)

        
        if "_commit_hash" in output:
            del output["_commit_hash"]
        if "_original_object_hash" in output:
            del output["_original_object_hash"]

        
        output["transformers_version"] = __version__

        self.dict_torch_dtype_to_str(output)
        return output

    def to_json_string(self, use_diff: bool = True, ignore_metadata: bool = False) -> str:
        
        if use_diff is True:
            config_dict = self.to_diff_dict()
        else:
            config_dict = self.to_dict()

        if ignore_metadata:
            for metadata_field in METADATA_FIELDS:
                config_dict.pop(metadata_field, None)

        return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"

    def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):
        
        with open(json_file_path, "w", encoding="utf-8") as writer:
            writer.write(self.to_json_string(use_diff=use_diff))

    @classmethod
    def from_model_config(cls, model_config: PretrainedConfig) -> "GenerationConfig":
        
        config_dict = model_config.to_dict()
        config_dict.pop("_from_model_config", None)
        config = cls.from_dict(config_dict, return_unused_kwargs=False, _from_model_config=True)

        
        
        for decoder_name in ("decoder", "generator", "text_config"):
            if decoder_name in config_dict:
                default_generation_config = GenerationConfig()
                decoder_config = config_dict[decoder_name]
                for attr in config.to_dict().keys():
                    if attr in decoder_config and getattr(config, attr) == getattr(default_generation_config, attr):
                        setattr(config, attr, decoder_config[attr])

        config._original_object_hash = hash(config)  
        return config

    def update(self, **kwargs):
        
        to_remove = []
        for key, value in kwargs.items():
            if hasattr(self, key):
                setattr(self, key, value)
                to_remove.append(key)

        
        unused_kwargs = {key: value for key, value in kwargs.items() if key not in to_remove}
        return unused_kwargs