codingutf8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license class logitsprocessor abstract base class for all logit warpers that can be applied during generation with multinomial sampling addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor raise notimplementederror fself class is an abstract class only classes inheriting this class can be called class logitsprocessorlistlist def callself inputids torch longtensor scores torch floattensor kwargs torch floattensor r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary what are input ids glossaryinputids scores torch floattensor of shape batchsize config vocabsize prediction scores of a language modeling head these can be logits for each vocabulary when not using beam search or log softmax for each vocabulary token when using beam search kwargs dictstr any optional additional kwargs that are specific to a logits processor return torch floattensor of shape batchsize config vocabsize the processed prediction scores def initself minlength int eostokenid unionint listint if not isinstanceminlength int or minlength 0 raise valueerrorfminlength has to be a nonnegative integer but is minlength if isinstanceeostokenid int eostokenid eostokenid if not allisinstancei int for i in eostokenid or anyi 0 for i in eostokenid logger warningfeostokenid has to be a list of positive integers but is eostokenid self minlength minlength self eostokenid eostokenid addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor curlen inputids shape1 if curlen self minlength for i in self eostokenid scores i floatinf return scores class minnewtokenslengthlogitsprocessorlogitsprocessor r logitsprocessor enforcing a minlength of new tokens by setting eos endofsequence token probability to 0 note that for decoderonly models such as llama2 minlength will compute the length of prompt newly generated tokens whereas for other models it will behave as minnewtokens that is taking only into account the newly generated ones args promptlengthtoskip int the input tokens length not a valid argument when used with generate as it will automatically assign the input length minnewtokens int the minimum new tokens length below which the score of eostokenid is set to floatinf eostokenid unionint listint the id of the endofsequence token optionally use a list to set multiple endofsequence tokens examples python from transformers import autotokenizer automodelforcausallm tokenizer autotokenizer frompretraineddistilgpt2 model automodelforcausallm frompretraineddistilgpt2 model config padtokenid model config eostokenid inputs tokenizerhugging face company is returntensorspt if the maximum length default 20 is smaller than the minimum length constraint the latter is ignored outputs model generateinputs minnewtokens30 printtokenizer decodeoutputs0 skipspecialtokenstrue hugging face company is a company that has been working on a new product for the past year for testing purposes let s set eostoken to company the first generated token this will make generation end there outputs model generateinputs eostokenid1664 printtokenizer decodeoutputs0 skipspecialtokenstrue hugging face company is a company increasing minnewtokens will make generation ignore occurences company eos token before the minimum length condition is honored outputs model generateinputs minnewtokens2 eostokenid1664 printtokenizer decodeoutputs0 skipspecialtokenstrue hugging face company is a new company def initself temperature float if not isinstancetemperature float or not temperature 0 exceptmsg ftemperature temperature has to be a strictly positive float otherwise your next token scores will be invalid if isinstancetemperature float and temperature 0 0 exceptmsg if you re looking for greedy decoding strategies set dosamplefalse raise valueerrorexceptmsg self temperature temperature addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor scores scores self temperature return scores class repetitionpenaltylogitsprocessorlogitsprocessor r logitsprocessor that prevents the repetition of previous tokens through an exponential penalty this technique shares some similarities with coverage mechanisms and other aimed at reducing repetition during the text generation process the probability distribution for the next token is determined using a formula that incorporates token scores based on their occurrence in the generated sequence tokens with higher scores are more likely to be selected the formula can be seen in the original paperhttps arxiv orgpdf1909 05858 pdf according to the paper a penalty of around 1 2 yields a good balance between truthful generation and lack of repetition this technique can also be used to reward and thus encourage repetition in a similar manner to penalize and reduce repetition use penalty values above 1 0 where a higher value penalizes more strongly to reward and encourage repetition use penalty values between 0 0 and 1 0 where a lower value rewards more strongly args penalty float the parameter for repetition penalty 1 0 means no penalty above 1 0 penalizes previously generated tokens between 0 0 and 1 0 rewards previously generated tokens see this paperhttps arxiv orgpdf1909 05858 pdf for more details examples py from transformers import autotokenizer automodelforcausallm initializing the model and tokenizer for it model automodelforcausallm frompretraineddistilgpt2 tokenizer autotokenizer frompretraineddistilgpt2 inputs tokenizeri m not going to returntensorspt this shows a normal generate without any specific parameters summaryids model generateinputs printtokenizer batchdecodesummaryids skipspecialtokenstrue0 i m not going to be able to do that i m going to be able to do that this generates a penalty for repeated tokens penalizedids model generateinputs repetitionpenalty1 1 printtokenizer batchdecodepenalizedids skipspecialtokenstrue0 i m not going to be able to do that i ll just have to go out and play if score 0 then repetition penalty has to be multiplied to reduce the token probabilities def initself penalty float encoderinputids torch longtensor if not isinstancepenalty float or not penalty 0 raise valueerrorfpenalty has to be a strictly positive float but is penalty self penalty 1 penalty self encoderinputids encoderinputids addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor score torch gatherscores 1 self encoderinputids if score 0 then hallucination penalty has to be multiplied to increase the token probabilities score torch wherescore 0 score self penalty score self penalty scores scatter1 self encoderinputids score return scores class topplogitswarperlogitswarper def initself topp float filtervalue float floatinf mintokenstokeep int 1 topp floattopp if topp 0 or topp 1 0 raise valueerrorftopp has to be a float 0 and 1 but is topp if not isinstancemintokenstokeep int or mintokenstokeep 1 raise valueerrorfmintokenstokeep has to be a positive integer but is mintokenstokeep self topp topp self filtervalue filtervalue self mintokenstokeep mintokenstokeep addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor sortedlogits sortedindices torch sortscores descendingfalse cumulativeprobs sortedlogits softmaxdim1 cumsumdim1 remove tokens with cumulative topp above the threshold token with 0 are kept sortedindicestoremove cumulativeprobs 1 self topp keep at least mintokenstokeep sortedindicestoremove self mintokenstokeep 0 scatter sorted tensors to original indexing indicestoremove sortedindicestoremove scatter1 sortedindices sortedindicestoremove scores scores maskedfillindicestoremove self filtervalue return scores class topklogitswarperlogitswarper r logitswarper that performs topk i e restricting to the k highest probability elements args topk int the number of highest probability vocabulary tokens to keep for topkfiltering filtervalue float optional defaults to inf all filtered values will be set to this float value mintokenstokeep int optional defaults to 1 minimum number of tokens that cannot be filtered remove all tokens with a probability less than the last token of the topk def initself mass float 0 9 filtervalue float floatinf mintokenstokeep int 1 mass floatmass if not mass 0 and mass 1 raise valueerrorftypicalp has to be a float 0 and 1 but is mass if not isinstancemintokenstokeep int or mintokenstokeep 1 raise valueerrorfmintokenstokeep has to be a positive integer but is mintokenstokeep self filtervalue filtervalue self mass mass self mintokenstokeep mintokenstokeep addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor calculate entropy normalized torch nn functional logsoftmaxscores dim1 p torch expnormalized ent normalized p nansum1 keepdimtrue shift and sort shiftedscores torch absnormalized ent sortedscores sortedindices torch sortshiftedscores descendingfalse sortedlogits scores gather1 sortedindices cumulativeprobs sortedlogits softmaxdim1 cumsumdim1 remove tokens with cumulative mass above the threshold lastind cumulativeprobs self mass sumdim1 lastind clampmaxsortedscores shape1 1 sortedindicestoremove sortedscores sortedscores gather1 lastind view1 1 sortedindicestoremove self mintokenstokeep 0 indicestoremove sortedindicestoremove scatter1 sortedindices sortedindicestoremove scores scores maskedfillindicestoremove self filtervalue return scores class epsilonlogitswarperlogitswarper r logitswarper that performs epsilonsampling i e restricting to tokens with prob epsilon takes the largest mintokenstokeep tokens if no tokens satisfy this constraint see truncation sampling as language model desmoothinghttps arxiv orgabs2210 15191 for more information args epsilon float if set to 0 only the most tokens with probabilities epsilon or higher are kept for generation filtervalue float optional defaults to inf all filtered values will be set to this float value mintokenstokeep int optional defaults to 1 minimum number of tokens that cannot be filtered examples python from transformers import autotokenizer automodelforcausallm setseed setseed0 model automodelforcausallm frompretraineddistilgpt2 tokenizer autotokenizer frompretraineddistilgpt2 inputs tokenizera sequence 1 2 returntensorspt with sampling the output is unexpected sometimes too unexpected outputs model generateinputs dosampletrue printtokenizer batchdecodeoutputs skipspecialtokenstrue0 a sequence 1 2 0 2 2 2 2 2 2 with epsilon sampling the output gets restricted to highprobability tokens note that this is similar to top p sampling which restricts tokens based on their cumulative probability pro tip the paper recomends using epsiloncutoff values between 3e4 and 9e4 outputs model generateinputs dosampletrue epsiloncutoff0 1 printtokenizer batchdecodeoutputs skipspecialtokenstrue0 a sequence 1 2 3 4 5 6 7 8 9 determine which indices to remove keep the words with the mintokenstokeep highest probabilities def initself epsilon float filtervalue float floatinf mintokenstokeep int 1 epsilon floatepsilon if epsilon 0 or epsilon 1 raise valueerrorfetacutoff has to be a float 0 and 1 but is epsilon mintokenstokeep intmintokenstokeep if mintokenstokeep 1 raise valueerror fmintokenstokeep has to be a strictly positive integer but is mintokenstokeep self epsilon torch tensorepsilon self filtervalue filtervalue self mintokenstokeep mintokenstokeep addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor calculate the adaptive cutoff probabilities scores softmaxdim1 entropy torch distributions categoricallogitsscores entropy eta torch minself epsilon torch sqrtself epsilon torch expentropy none indicestoremove probabilities eta keep the words with the mintokenstokeep highest probabilities topk minself mintokenstokeep scores size1 safety check indicestoremove indicestoremove scores torch topkscores topk0 1 none scores scores maskedfillindicestoremove self filtervalue return scores def getngramsngramsize int previnputids torch tensor numhypos int initialize an empty list of dictionaries one for each hypothesis index in the range of numhypos generatedngrams for in rangenumhypos for idx in rangenumhypos gentokens previnputidsidx tolist generatedngram generatedngramsidx loop through each ngram of size ngramsize in the list of tokens gentokens for ngram in zipgentokensi for i in rangengramsize prevngramtuple tuplengram 1 generatedngramprevngramtuple generatedngram getprevngramtuple ngram1 return generatedngrams def getgeneratedngramsbannedngrams previnputids ngramsize curlen before decoding the next token prevent decoding of ngrams that have already appeared startidx curlen 1 ngramsize ngramidx tupleprevinputidsstartidx curlen tolist return bannedngrams getngramidx def calcbannedngramtokens ngramsize int previnputids torch tensor numhypos int curlen int listiterableint return no banned tokens if we haven t generated norepeatngramsize tokens yet def initself ngramsize int if not isinstancengramsize int or ngramsize 0 raise valueerrorfngramsize has to be a strictly positive integer but is ngramsize self ngramsize ngramsize addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor numbatchhypotheses scores shape0 curlen inputids shape1 bannedbatchtokens calcbannedngramtokensself ngramsize inputids numbatchhypotheses curlen for i bannedtokens in enumeratebannedbatchtokens scoresi bannedtokens floatinf return scores class encodernorepeatngramlogitsprocessorlogitsprocessor r logitsprocessor that enforces no repetition of encoder input ids ngrams for the decoder ids see parlaihttps github comfacebookresearchparlaiblobmasterparlaicoretorchgeneratoragent pyl1350 args encoderngramsize int all ngrams of size ngramsize can only occur within the encoder input ids encoderinputids int the encoderinputids that should not be repeated within the decoder ids b x numbeams logitsprocessor that applies an additive bias on sequences the bias is applied to the last token of a sequence when the next generated token can complete it consequently to take the most of biasing sequences with more than one token consider using beam methods to gracefully work around partially completed sequences that have a negative bias and applying the bias to their prefixes to ensure the bias is applied earlier tip in order to get the token ids of the sequences that you want to bias make sure to set addprefixspacetrue when initializing the tokenizer and use tokenizerbadwords addspecialtokensfalse inputids the addprefixspace argument is only supported for some slow tokenizers as fast tokenizers prefixing behaviours come from pre tokenizers read more herehttps huggingface codocstokenizersapipretokenizers tip args sequencebias dicttupleint float dictionary that maps a sequence of tokens to its bias term positive biases increase the odds of the sequence being selected while negative biases do the opposite if a sequence has a length of 1 its bias will always be applied otherwise the bias will only be applied if the sequence in question is about to be completed in the token selection step after this processor is applied examples python from transformers import autotokenizer automodelforcausallm model automodelforcausallm frompretrainedgpt2 tokenizer autotokenizer frompretrainedgpt2 inputs tokenizerthe full name of donald is donald returntensorspt summaryids model generateinputsinputids maxnewtokens4 printtokenizer batchdecodesummaryids skipspecialtokenstrue0 the full name of donald is donald j trump jr now let s control generation through a bias please note that the tokenizer is initialized differently tokenizerwithprefixspace autotokenizer frompretrainedgpt2 addprefixspacetrue def gettokensastupleword return tupletokenizerwithprefixspaceword addspecialtokensfalse inputids0 if we add a negative bias without beam search it may become stuck in a prefix without good continuations sequencebias gettokensastupletrump 10 0 biasedids model generateinputsinputids maxnewtokens4 sequencebiassequencebias printtokenizer batchdecodebiasedids skipspecialtokenstrue0 the full name of donald is donald j donald biasedids model generateinputsinputids maxnewtokens4 numbeams4 sequencebiassequencebias printtokenizer batchdecodebiasedids skipspecialtokenstrue0 the full name of donald is donald rumsfeld we can also add a positive bias to nudge the model towards specific tokens or continuations sequencebias gettokensastupledonald duck 10 0 biasedids model generateinputsinputids maxnewtokens4 numbeams4 sequencebiassequencebias printtokenizer batchdecodebiasedids skipspecialtokenstrue0 the full name of donald is donald duck bias variables that will be populated on the first call for retrocompatibility purposes the vocabulary size is infered in the first usage which inhibits initializing here 1 prepares the bias tensors this is only needed the first time the logit processor is called 2 prepares an empty bias to add 3 include the bias from length 1 4 include the bias from length 1 after determining which biased sequences may be completed 5 apply the bias to the scores check biased tokens out of bounds precompute the bias tensors to be applied sequences of length 1 are kept separately as they can be applied with simpler logic logitsprocessor that enforces that specified sequences will never be selected tip in order to get the token ids of the words that should not appear in the generated text make sure to set addprefixspacetrue when initializing the tokenizer and use tokenizerbadwords addspecialtokensfalse inputids the addprefixspace argument is only supported for some slow tokenizers as fast tokenizers prefixing behaviours come from pre tokenizers read more herehttps huggingface codocstokenizersapipretokenizers tip args badwordsids listlistint list of list of token ids that are not allowed to be generated eostokenid unionint listint the id of the endofsequence token optionally use a list to set multiple endofsequence tokens examples python from transformers import autotokenizer automodelforcausallm model automodelforcausallm frompretrainedgpt2 tokenizer autotokenizer frompretrainedgpt2 inputs tokenizerin a word the cake is a returntensorspt outputids model generateinputsinputids maxnewtokens5 padtokenidtokenizer eostokenid printtokenizer batchdecodeoutputids skipspecialtokenstrue0 in a word the cake is a bit of a mess now let s take the bad words out please note that the tokenizer is initialized differently tokenizerwithprefixspace autotokenizer frompretrainedgpt2 addprefixspacetrue def gettokensaslistwordlist converts a sequence of words into a list of tokens tokenslist for word in wordlist tokenizedword tokenizerwithprefixspaceword addspecialtokensfalse inputids0 tokenslist appendtokenizedword return tokenslist badwordsids gettokensaslistwordlistmess outputids model generate inputsinputids maxnewtokens5 badwordsidsbadwordsids padtokenidtokenizer eostokenid printtokenizer batchdecodeoutputids skipspecialtokenstrue0 in a word the cake is a bit of a surprise filter eos token from badwordsids forbidding a sequence is equivalent to setting its bias to inf def initself prefixallowedtokensfn callableint torch tensor listint numbeams int self prefixallowedtokensfn prefixallowedtokensfn self numbeams numbeams addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor mask torch fulllikescores math inf for batchid beamsent in enumerateinputids view1 self numbeams inputids shape1 for beamid sent in enumeratebeamsent maskbatchid self numbeams beamid self prefixallowedtokensfnbatchid sent 0 return scores mask class hammingdiversitylogitsprocessorlogitsprocessor r logitsprocessor that enforces diverse beam search note that this logits processor is only effective for pretrainedmodel groupbeamsearch see diverse beam search decoding diverse solutions from neural sequence modelshttps arxiv orgpdf1610 02424 pdf for more details tip diverse beam search can be particularly useful in scenarios where a variety of different outputs is desired rather than multiple similar sequences it allows the model to explore different generation paths and provides a broader coverage of possible outputs tip tip warningtrue this logits processor can be resourceintensive especially when using large models or long sequences tip traditional beam search often generates very similar sequences across different beams hammingdiversitylogitsprocessor addresses this by penalizing beams that generate tokens already chosen by other beams in the same time step how it works grouping beams beams are divided into groups each group selects tokens independently of the others penalizing repeated tokens if a beam in a group selects a token already chosen by another group in the same step a penalty is applied to that token s score promoting diversity this penalty discourages beams within a group from selecting the same tokens as beams in other groups benefits diverse outputs produces a variety of different sequences exploration allows the model to explore different paths args diversitypenalty float this value is subtracted from a beam s score if it generates a token same as any beam from other group at a particular time note that diversitypenalty is only effective if group beam search is enabled the penalty applied to a beam s score when it generates a token that has already been chosen by another beam within the same group during the same time step a higher diversitypenalty will enforce greater diversity among the beams making it less likely for multiple beams to choose the same token conversely a lower penalty will allow beams to more freely choose similar tokens adjusting this value can help strike a balance between diversity and natural likelihood numbeams int number of beams used for group beam search beam search is a method used that maintains beams or multiple hypotheses at each step expanding each one and keeping the topscoring sequences a higher numbeams will explore more potential sequences this can increase chances of finding a highquality output but also increases computational cost numbeamgroups int number of groups to divide numbeams into in order to ensure diversity among different groups of beams each group of beams will operate independently selecting tokens without considering the choices of other groups this division promotes diversity by ensuring that beams within different groups explore different paths for instance if numbeams is 6 and numbeamgroups is 2 there will be 2 groups each containing 3 beams the choice of numbeamgroups should be made considering the desired level of output diversity and the total number of beams see this paperhttps arxiv orgpdf1610 02424 pdf for more details examples python from transformers import autotokenizer automodelforseq2seqlm import torch initialize the model and tokenizer tokenizer autotokenizer frompretrainedt5base model automodelforseq2seqlm frompretrainedt5base a long text about the solar system text the solar system is a gravitationally bound system comprising the sun and the objects that orbit it either directly or indirectly of the objects that orbit the sun directly the largest are the eight planets with the remainder being smaller objects such as the five dwarf planets and small solar system bodies the solar system formed 4 6 billion years ago from the gravitational collapse of a giant interstellar molecular cloud inputs tokenizersummarize text returntensorspt generate diverse summary outputsdiverse model generate inputs numbeamgroups2 diversitypenalty10 0 maxlength100 numbeams4 numreturnsequences2 summariesdiverse tokenizer batchdecodeoutputsdiverse skipspecialtokenstrue generate nondiverse summary outputsnondiverse model generate inputs maxlength100 numbeams4 numreturnsequences2 summarynondiverse tokenizer batchdecodeoutputsnondiverse skipspecialtokenstrue with diversitypenalty the resulting beams are much more diverse printsummarynondiverse the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets printsummariesdiverse the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the rest of the objects are smaller objects such as the five dwarf planets and small solar system bodies hamming diversity penalise using same token in current group which was used in previous groups at the same time step batchsize currenttokens shape0 self numbeams groupstartidx beamgroupidx self numsubbeams groupendidx mingroupstartidx self numsubbeams self numbeams groupsize groupendidx groupstartidx vocabsize scores shape1 if groupstartidx 0 return scores for batchidx in rangebatchsize predicted tokens of last time step of previous groups previousgrouptokens currenttokens batchidx self numbeams batchidx self numbeams groupstartidx tokenfrequency torch bincountpreviousgrouptokens minlengthvocabsize toscores device scoresbatchidx groupsize batchidx 1 groupsize self diversitypenalty tokenfrequency return scores class forcedbostokenlogitsprocessorlogitsprocessor r logitsprocessor that enforces the specified token as the first generated token args bostokenid int the id of the token to force as the first generated token def initself maxlength int eostokenid unionint listint self maxlength maxlength if isinstanceeostokenid int eostokenid eostokenid self eostokenid eostokenid addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor curlen inputids shape1 if curlen self maxlength 1 numtokens scores shape1 scores i for i in rangenumtokens if i not in self eostokenid floatinf for i in self eostokenid scores i 0 return scores class infnanremovelogitsprocessorlogitsprocessor r logitsprocessor that removes all nan and inf values to avoid the generation method to fail note that using the logits processor should only be used if necessary since it can slow down the generation method set all nan values to 0 0 set all inf values to maxmin possible value def init self exponentialdecaylengthpenalty tupleint float eostokenid unionint listint inputidsseqlength int self regulationstart exponentialdecaylengthpenalty0 inputidsseqlength self regulationfactor exponentialdecaylengthpenalty1 if isinstanceeostokenid int eostokenid eostokenid self eostokenid eostokenid addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor curlen inputids shape1 if curlen self regulationstart for i in self eostokenid penaltyidx curlen self regulationstart to support negative logits we compute the penalty of the absolute value and add to the original logit scores i scores i torch absscores i powself regulationfactor penaltyidx 1 return scores class logitnormalizationlogitsprocessor logitswarper r logitswarper and logitsprocessor for normalizing the scores using logsoftmax it s important to normalize the scores during beam search after applying the logits processors or warpers since the search algorithm used in this library doesn t do it it only does it before but they may need renormalization but it still supposes that the scores are normalized when comparing the hypotheses def initself beginsuppresstokens beginindex self beginsuppresstokens listbeginsuppresstokens self beginindex beginindex addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor if inputids shape1 self beginindex scores self beginsuppresstokens floatinf return scores class suppresstokenslogitsprocessorlogitsprocessor rthis processor can be used to suppress a list of tokens the processor will set their log probs to inf so that they are not sampled def initself suppresstokens self suppresstokens listsuppresstokens addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor scores self suppresstokens floatinf return scores class forcetokenslogitsprocessorlogitsprocessor rthis processor takes a list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling the processor will set their log probs to inf so that they are sampled at their corresponding index def initself forcetokenmap listlistint self forcetokenmap dictforcetokenmap addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor generationidx inputids shape1 currenttoken self forcetokenmap getgenerationidx none if currenttoken is not none scores floatinf scores currenttoken 0 return scores class whispertimestamplogitsprocessorlogitsprocessor r logitsprocessor that modifies the logits for the generation of timestamps in the transcription when the input tokens are at a specific threshold the processor sets the scores to negative infinity the processor makes sure that timestamp tokens appear in pairs by masking out the logits that would break this pairing pattern this is done to maintain the consistency and structure of generated timestamps it also ensures that when the predicted probability of sampling any of the timestamp token is greater than any individual nontimestamp token those nontimestamp logits are set to negative infinity this is done to ensure the generation of timestamps over other potential tokens see the paperhttps arxiv orgabs2212 04356 for more information args generateconfig generateconfig the generate config used to generate the output the following parameters are required eostokenid int optional defaults to 50257 the id of the endofsequence token notimestampstokenid int optional defaults to 50363 the id of the notimestamps token maxinitialtimestampindex int optional defaults to 1 used to set the maximum value of the initial timestamp this is used to prevent the model from predicting timestamps that are too far in the future detecttimestampfromlogprob bool optional whether timestamps can be predicted from logprobs over all timestamps examples python import torch from transformers import autoprocessor whisperforconditionalgeneration generationconfig from datasets import loaddataset processor autoprocessor frompretrainedopenaiwhispertiny en model whisperforconditionalgeneration frompretrainedopenaiwhispertiny en ds loaddatasethfinternaltestinglibrispeechasrdummy clean splitvalidation inputs processords3audioarray returntensorspt inputfeatures inputs inputfeatures displaying timestamps generatedids model generateinputsinputfeatures returntimestampstrue transcription processor batchdecodegeneratedids skipspecialtokensfalse0 printtranscription transcription transcription startoftranscript0 00 he has grave doubts whether sir frederick layton s work is really greek after all and can6 446 44 discover in it but little of rocky ithaca 9 44endoftext no timestamps change eos this allows the user to select a specific token to terminate the sequence on in this case it s the word can460 model generationconfig eostokenid 460 generatedids model generateinputsinputfeatures returntimestampsfalse transcription processor batchdecodegeneratedids skipspecialtokenstrue0 printtranscription transcription transcription he has grave doubts whether sir frederick layton s work is really greek after all and can this variable is mostly just used for testing suppress notimestamps which is handled by withouttimestamps timestamps have to appear in pairs except directly before eostoken mask logits accordingly timestamps shouldn t decrease forbid timestamp tokens smaller than the last the following lines of code are copied from https github comopenaiwhisperpull914filesr1137085090 avoid to emit 0 00 again apply the maxinitialtimestamp option if sum of probability over timestamps is above any other token sample timestamp def initself guidancescale if guidancescale 1 self guidancescale guidancescale else raise valueerror require guidance scale 1 to use the classifier free guidance processor got guidance scale fguidancescale addstartdocstringslogitsprocessorinputsdocstring def callself inputids torch longtensor scores torch floattensor torch floattensor simple check to make sure we have compatible batch sizes between our logits scores cond uncond and input ids cond only if scores shape0 2 inputids shape0 raise valueerror flogits should have twice the batch size of the input ids the first half of batches corresponding to fthe conditional inputs and the second half of batches corresponding to the unconditional inputs got fbatch size scores shape0 for the logits and inputids shape0 for the input ids unguidedbsz scores shape0 2 condlogits uncondlogits scores splitunguidedbsz dim0 scores uncondlogits condlogits uncondlogits self guidancescale return scores class alternatingcodebookslogitsprocessorlogitsprocessor r logitsprocessor enforcing alternated generation between the two codebooks of bark s fine submodel args inputstartlen int the length of the initial input sequence semanticvocabsize int vocabulary size of the semantic part i e number of tokens associated to the semantic vocabulary codebooksize int number of tokens associated to the codebook even first codebook odd second codebook def init self guidancescale float model unconditionalids optionaltorch longtensor none unconditionalattentionmask optionaltorch longtensor none usecache optionalbool true self guidancescale guidancescale self model model self unconditionalcontext inputids unconditionalids attentionmask unconditionalattentionmask usecache usecache pastkeyvalues none firstpass true def getunconditionallogitsself inputids if self unconditionalcontextfirstpass if self unconditionalcontextinputids is none self unconditionalcontextinputids inputids 1 if self unconditionalcontextattentionmask is none self unconditionalcontextattentionmask torch oneslike self unconditionalcontextinputids dtypetorch long inputids self unconditionalcontextinputids attentionmask self unconditionalcontextattentionmask self unconditionalcontextfirstpass false else attentionmask torch cat self unconditionalcontextattentionmask torch oneslikeinputids 1 dtypetorch long dim1 if not self unconditionalcontextusecache inputids torch catself unconditionalcontextinputids inputids 1 dim1 else inputids inputids 1 self unconditionalcontextinputids inputids self unconditionalcontextattentionmask attentionmask out self model inputids attentionmaskattentionmask usecacheself unconditionalcontextusecache pastkeyvaluesself unconditionalcontextpastkeyvalues self unconditionalcontextpastkeyvalues out getpastkeyvalues none return out logits def callself inputids scores scores torch nn functional logsoftmaxscores dim1 if self guidancescale 1 return scores logits self getunconditionallogitsinputids unconditionallogits torch nn functional logsoftmaxlogits 1 dim1 out self guidancescale scores unconditionallogits unconditionallogits return out class barkeosprioritizerlogitsprocessorlogitsprocessor rthis processor ensures that the eos token is selected if its probability is greater than the mineosp args eostokenid unionint listint the id of the endofsequence token optionally use a list to set multiple endofsequence tokens mineosp float optional minimum end of speech threshold create scores full of inf except for the eostokenid coding utf 8 2020 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary what are input ids glossary input ids scores torch floattensor of shape batch_size config vocab_size prediction scores of a language modeling head these can be logits for each vocabulary when not using beam search or log softmax for each vocabulary token when using beam search return torch floattensor of shape batch_size config vocab_size the processed prediction scores abstract base class for all logit processors that can be applied during generation abstract base class for all logit warpers that can be applied during generation with multinomial sampling this class can be used to create a list of logitsprocessor or logitswarper to subsequently process a scores input tensor this class inherits from list and adds a specific __call__ method to apply each logitsprocessor or logitswarper to the inputs args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary what are input ids glossary input ids scores torch floattensor of shape batch_size config vocab_size prediction scores of a language modeling head these can be logits for each vocabulary when not using beam search or log softmax for each vocabulary token when using beam search kwargs dict str any optional additional kwargs that are specific to a logits processor return torch floattensor of shape batch_size config vocab_size the processed prediction scores logitsprocessor enforcing a min length by setting eos probability to 0 args min_length int the minimum length below which the score of eos_token_id is set to float inf eos_token_id union int list int the id of the end of sequence token optionally use a list to set multiple end of sequence tokens logitsprocessor enforcing a min length of new tokens by setting eos end of sequence token probability to 0 note that for decoder only models such as llama2 min_length will compute the length of prompt newly generated tokens whereas for other models it will behave as min_new_tokens that is taking only into account the newly generated ones args prompt_length_to_skip int the input tokens length not a valid argument when used with generate as it will automatically assign the input length min_new_tokens int the minimum new tokens length below which the score of eos_token_id is set to float inf eos_token_id union int list int the id of the end of sequence token optionally use a list to set multiple end of sequence tokens examples python from transformers import autotokenizer automodelforcausallm tokenizer autotokenizer from_pretrained distilgpt2 model automodelforcausallm from_pretrained distilgpt2 model config pad_token_id model config eos_token_id inputs tokenizer hugging face company is return_tensors pt if the maximum length default 20 is smaller than the minimum length constraint the latter is ignored outputs model generate inputs min_new_tokens 30 print tokenizer decode outputs 0 skip_special_tokens true hugging face company is a company that has been working on a new product for the past year for testing purposes let s set eos_token to company the first generated token this will make generation end there outputs model generate inputs eos_token_id 1664 print tokenizer decode outputs 0 skip_special_tokens true hugging face company is a company increasing min_new_tokens will make generation ignore occurences company eos token before the minimum length condition is honored outputs model generate inputs min_new_tokens 2 eos_token_id 1664 print tokenizer decode outputs 0 skip_special_tokens true hugging face company is a new company logitswarper for temperature exponential scaling output probability distribution which effectively means that it can control the randomness of the predicted tokens tip make sure that do_sample true is included in the generate arguments otherwise the temperature value won t have any effect tip args temperature float strictly positive float value used to modulate the logits distribution a value smaller than 1 decreases randomness and vice versa with 0 being equivalent to shifting all probability mass to the most likely token examples python import torch from transformers import autotokenizer automodelforcausallm set_seed set_seed 0 for reproducibility tokenizer autotokenizer from_pretrained gpt2 model automodelforcausallm from_pretrained gpt2 model config pad_token_id model config eos_token_id inputs tokenizer hugging face company is return_tensors pt with temperature 1 0 the default we consistently get random outputs due to random sampling generate_kwargs max_new_tokens 10 do_sample true temperature 1 0 num_return_sequences 2 outputs model generate inputs generate_kwargs print tokenizer batch_decode outputs skip_special_tokens true hugging face company is a joint venture between geo group one of hugging face company is not an exact science but what we believe does however with temperature close to 0 it approximates greedy decoding strategies invariant generate_kwargs temperature 0 0001 outputs model generate inputs generate_kwargs print tokenizer batch_decode outputs skip_special_tokens true hugging face company is a company that has been around for over 20 years hugging face company is a company that has been around for over 20 years logitsprocessor that prevents the repetition of previous tokens through an exponential penalty this technique shares some similarities with coverage mechanisms and other aimed at reducing repetition during the text generation process the probability distribution for the next token is determined using a formula that incorporates token scores based on their occurrence in the generated sequence tokens with higher scores are more likely to be selected the formula can be seen in the original paper https arxiv org pdf 1909 05858 pdf according to the paper a penalty of around 1 2 yields a good balance between truthful generation and lack of repetition this technique can also be used to reward and thus encourage repetition in a similar manner to penalize and reduce repetition use penalty values above 1 0 where a higher value penalizes more strongly to reward and encourage repetition use penalty values between 0 0 and 1 0 where a lower value rewards more strongly args penalty float the parameter for repetition penalty 1 0 means no penalty above 1 0 penalizes previously generated tokens between 0 0 and 1 0 rewards previously generated tokens see this paper https arxiv org pdf 1909 05858 pdf for more details examples py from transformers import autotokenizer automodelforcausallm initializing the model and tokenizer for it model automodelforcausallm from_pretrained distilgpt2 tokenizer autotokenizer from_pretrained distilgpt2 inputs tokenizer i m not going to return_tensors pt this shows a normal generate without any specific parameters summary_ids model generate inputs print tokenizer batch_decode summary_ids skip_special_tokens true 0 i m not going to be able to do that i m going to be able to do that this generates a penalty for repeated tokens penalized_ids model generate inputs repetition_penalty 1 1 print tokenizer batch_decode penalized_ids skip_special_tokens true 0 i m not going to be able to do that i ll just have to go out and play if score 0 then repetition penalty has to be multiplied to reduce the token probabilities logitsprocessor that avoids hallucination by boosting the probabilities of tokens found within the original input this technique can also be used to reward and thus encourage hallucination or creativity in a similar manner to penalize and reduce hallucination use penalty values above 1 0 where a higher value penalizes more strongly to reward and encourage hallucination use penalty values between 0 0 and 1 0 where a lower value rewards more strongly args penalty float the parameter for hallucination penalty 1 0 means no penalty above 1 0 penalizes hallucination between 0 0 and 1 0 rewards hallucination encoder_input_ids torch longtensor the encoder_input_ids that should be repeated within the decoder ids if score 0 then hallucination penalty has to be multiplied to increase the token probabilities logitswarper that performs top p i e restricting to top tokens summing to prob_cut_off prob_cut_off args top_p float if set to 1 only the smallest set of most probable tokens with probabilities that add up to top_p or higher are kept for generation filter_value float optional defaults to inf all filtered values will be set to this float value min_tokens_to_keep int optional defaults to 1 minimum number of tokens that cannot be filtered examples python from transformers import autotokenizer automodelforcausallm set_seed set_seed 0 model automodelforcausallm from_pretrained distilgpt2 tokenizer autotokenizer from_pretrained distilgpt2 inputs tokenizer a sequence 1 2 return_tensors pt with sampling the output is unexpected sometimes too unexpected outputs model generate inputs do_sample true print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 0 2 2 2 2 2 2 with top_p sampling the output gets restricted to high probability tokens pro tip in practice llms use top_p in the 0 9 0 95 range outputs model generate inputs do_sample true top_p 0 1 print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 3 4 5 6 7 8 9 remove tokens with cumulative top_p above the threshold token with 0 are kept keep at least min_tokens_to_keep scatter sorted tensors to original indexing logitswarper that performs top k i e restricting to the k highest probability elements args top_k int the number of highest probability vocabulary tokens to keep for top k filtering filter_value float optional defaults to inf all filtered values will be set to this float value min_tokens_to_keep int optional defaults to 1 minimum number of tokens that cannot be filtered safety check remove all tokens with a probability less than the last token of the top k logitswarper that performs typical decoding see typical decoding for natural language generation https arxiv org abs 2202 00666 for more information args mass float optional defaults to 0 9 value of typical_p between 0 and 1 inclusive defaults to 0 9 filter_value float optional defaults to inf all filtered values will be set to this float value min_tokens_to_keep int optional defaults to 1 minimum number of tokens that cannot be filtered calculate entropy shift and sort remove tokens with cumulative mass above the threshold logitswarper that performs epsilon sampling i e restricting to tokens with prob epsilon takes the largest min_tokens_to_keep tokens if no tokens satisfy this constraint see truncation sampling as language model desmoothing https arxiv org abs 2210 15191 for more information args epsilon float if set to 0 only the most tokens with probabilities epsilon or higher are kept for generation filter_value float optional defaults to inf all filtered values will be set to this float value min_tokens_to_keep int optional defaults to 1 minimum number of tokens that cannot be filtered examples python from transformers import autotokenizer automodelforcausallm set_seed set_seed 0 model automodelforcausallm from_pretrained distilgpt2 tokenizer autotokenizer from_pretrained distilgpt2 inputs tokenizer a sequence 1 2 return_tensors pt with sampling the output is unexpected sometimes too unexpected outputs model generate inputs do_sample true print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 0 2 2 2 2 2 2 with epsilon sampling the output gets restricted to high probability tokens note that this is similar to top p sampling which restricts tokens based on their cumulative probability pro tip the paper recomends using epsilon_cutoff values between 3e 4 and 9e 4 outputs model generate inputs do_sample true epsilon_cutoff 0 1 print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 3 4 5 6 7 8 9 determine which indices to remove keep the words with the min_tokens_to_keep highest probabilities safety check logitswarper that performs eta sampling a technique to filter out tokens with probabilities below a dynamic cutoff value eta which is calculated based on a combination of the hyperparameter epsilon and the entropy of the token probabilities i e eta min epsilon sqrt epsilon e entropy probabilities takes the largest min_tokens_to_keep tokens if no tokens satisfy this constraint it addresses the issue of poor quality in long samples of text generated by neural language models leading to more coherent and fluent text see truncation sampling as language model desmoothing https arxiv org abs 2210 15191 for more information note do_sample must be set to true for this logitswarper to work args epsilon float a float value in the range 0 1 hyperparameter used to calculate the dynamic cutoff value eta the suggested values from the paper ranges from 3e 4 to 4e 3 depending on the size of the model filter_value float optional defaults to inf all values that are found to be below the dynamic cutoff value eta are set to this float value this parameter is useful when logits need to be modified for very low probability tokens that should be excluded from generation entirely min_tokens_to_keep int optional defaults to 1 specifies the minimum number of tokens that must be kept for generation regardless of their probabilities for example if min_tokens_to_keep is set to 1 at least one token will always be kept for generation even if all tokens have probabilities below the cutoff eta examples python from transformers import autotokenizer automodelforcausallm set_seed set_seed 0 model automodelforcausallm from_pretrained distilgpt2 tokenizer autotokenizer from_pretrained distilgpt2 inputs tokenizer a sequence 1 2 return_tensors pt with sampling the output is unexpected sometimes too unexpected outputs model generate inputs do_sample true print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 0 2 2 2 2 2 2 with eta sampling the output gets restricted to high probability tokens you can see it as a dynamic form of epsilon sampling that adapts its cutoff probability based on the entropy high entropy lower cutoff pro tip the paper recomends using eta_cutoff values between 3e 4 to 4e 3 outputs model generate inputs do_sample true eta_cutoff 0 1 print tokenizer batch_decode outputs skip_special_tokens true 0 a sequence 1 2 3 4 5 6 7 8 9 calculate the adaptive cutoff keep the words with the min_tokens_to_keep highest probabilities safety check assume ngram_size 2 and prev_input_ids tensor 40 2883 2712 4346 the output of generated ngrams look like this 40 2883 2883 2712 2712 4346 args ngram_size int the number sequential tokens taken as a group which may only occur once before being banned prev_input_ids torch tensor generated token ids for the current hypothesis num_hypos int the number of hypotheses for which n grams need to be generated returns generated_ngrams dict dictionary of generated ngrams initialize an empty list of dictionaries one for each hypothesis index in the range of num_hypos loop through each n gram of size ngram_size in the list of tokens gen_tokens determines the banned tokens for the current hypothesis based on previously generated n grams args banned_ngrams dict a dictionary containing previously generated n grams for each hypothesis prev_input_ids torch tensor generated token ids for the current hypothesis ngram_size int the number sequential tokens taken as a group which may only occur once before being banned cur_len int the current length of the token sequences for which the n grams are being checked returns list of tokens that are banned before decoding the next token prevent decoding of ngrams that have already appeared copied from fairseq for no_repeat_ngram in beam_search return no banned tokens if we haven t generated no_repeat_ngram_size tokens yet n grams are groups of n consecutive words characters or tokens taken from a sequence of text given the sentence she runs fast the bi grams n 2 would be she runs and runs fast in text generation avoiding repetitions of word sequences provides a more diverse output this logitsprocessor enforces no repetition of n grams by setting the scores of banned tokens to negative infinity which eliminates those tokens from consideration when further processing the scores fairseq https github com pytorch fairseq blob a07cb6f40480928c9e0548b737aadd36ee66ac76 fairseq sequence_generator py l345 tip use n gram penalties with care for instance penalizing 2 grams bigrams in an article about the city of new york might lead to undesirable outcomes where the city s name appears only once in the entire text reference https huggingface co blog how to generate tip args ngram_size int all ngrams of size ngram_size can only occur once examples py from transformers import autotokenizer automodelforcausallm model automodelforcausallm from_pretrained distilgpt2 tokenizer autotokenizer from_pretrained distilgpt2 inputs tokenizer today i return_tensors pt output model generate inputs print tokenizer decode output 0 skip_special_tokens true today i m not sure if i m going to be able to do it now let s add ngram size using no_repeat_ngram_size this stops the repetitions i m in the output output model generate inputs no_repeat_ngram_size 2 print tokenizer decode output 0 skip_special_tokens true today i m not sure if i can get a better understanding of the nature of this issue logitsprocessor that enforces no repetition of encoder input ids n grams for the decoder ids see parlai https github com facebookresearch parlai blob master parlai core torch_generator_agent py l1350 args encoder_ngram_size int all ngrams of size ngram_size can only occur within the encoder input ids encoder_input_ids int the encoder_input_ids that should not be repeated within the decoder ids b x num_beams logitsprocessor that applies an additive bias on sequences the bias is applied to the last token of a sequence when the next generated token can complete it consequently to take the most of biasing sequences with more than one token consider using beam methods to gracefully work around partially completed sequences that have a negative bias and applying the bias to their prefixes to ensure the bias is applied earlier tip in order to get the token ids of the sequences that you want to bias make sure to set add_prefix_space true when initializing the tokenizer and use tokenizer bad_words add_special_tokens false input_ids the add_prefix_space argument is only supported for some slow tokenizers as fast tokenizers prefixing behaviours come from pre tokenizers read more here https huggingface co docs tokenizers api pre tokenizers tip args sequence_bias dict tuple int float dictionary that maps a sequence of tokens to its bias term positive biases increase the odds of the sequence being selected while negative biases do the opposite if a sequence has a length of 1 its bias will always be applied otherwise the bias will only be applied if the sequence in question is about to be completed in the token selection step after this processor is applied examples python from transformers import autotokenizer automodelforcausallm model automodelforcausallm from_pretrained gpt2 tokenizer autotokenizer from_pretrained gpt2 inputs tokenizer the full name of donald is donald return_tensors pt summary_ids model generate inputs input_ids max_new_tokens 4 print tokenizer batch_decode summary_ids skip_special_tokens true 0 the full name of donald is donald j trump jr now let s control generation through a bias please note that the tokenizer is initialized differently tokenizer_with_prefix_space autotokenizer from_pretrained gpt2 add_prefix_space true def get_tokens_as_tuple word return tuple tokenizer_with_prefix_space word add_special_tokens false input_ids 0 if we add a negative bias without beam search it may become stuck in a prefix without good continuations sequence_bias get_tokens_as_tuple trump 10 0 biased_ids model generate inputs input_ids max_new_tokens 4 sequence_bias sequence_bias print tokenizer batch_decode biased_ids skip_special_tokens true 0 the full name of donald is donald j donald biased_ids model generate inputs input_ids max_new_tokens 4 num_beams 4 sequence_bias sequence_bias print tokenizer batch_decode biased_ids skip_special_tokens true 0 the full name of donald is donald rumsfeld we can also add a positive bias to nudge the model towards specific tokens or continuations sequence_bias get_tokens_as_tuple donald duck 10 0 biased_ids model generate inputs input_ids max_new_tokens 4 num_beams 4 sequence_bias sequence_bias print tokenizer batch_decode biased_ids skip_special_tokens true 0 the full name of donald is donald duck bias variables that will be populated on the first call for retrocompatibility purposes the vocabulary size is infered in the first usage which inhibits initializing here 1 prepares the bias tensors this is only needed the first time the logit processor is called 2 prepares an empty bias to add 3 include the bias from length 1 4 include the bias from length 1 after determining which biased sequences may be completed the sequence is of length 1 already applied the sequence is longer than the context ignore 5 apply the bias to the scores check biased tokens out of bounds precompute the bias tensors to be applied sequences of length 1 are kept separately as they can be applied with simpler logic logitsprocessor that enforces that specified sequences will never be selected tip in order to get the token ids of the words that should not appear in the generated text make sure to set add_prefix_space true when initializing the tokenizer and use tokenizer bad_words add_special_tokens false input_ids the add_prefix_space argument is only supported for some slow tokenizers as fast tokenizers prefixing behaviours come from pre tokenizers read more here https huggingface co docs tokenizers api pre tokenizers tip args bad_words_ids list list int list of list of token ids that are not allowed to be generated eos_token_id union int list int the id of the end of sequence token optionally use a list to set multiple end of sequence tokens examples python from transformers import autotokenizer automodelforcausallm model automodelforcausallm from_pretrained gpt2 tokenizer autotokenizer from_pretrained gpt2 inputs tokenizer in a word the cake is a return_tensors pt output_ids model generate inputs input_ids max_new_tokens 5 pad_token_id tokenizer eos_token_id print tokenizer batch_decode output_ids skip_special_tokens true 0 in a word the cake is a bit of a mess now let s take the bad words out please note that the tokenizer is initialized differently tokenizer_with_prefix_space autotokenizer from_pretrained gpt2 add_prefix_space true def get_tokens_as_list word_list converts a sequence of words into a list of tokens tokens_list for word in word_list tokenized_word tokenizer_with_prefix_space word add_special_tokens false input_ids 0 tokens_list append tokenized_word return tokens_list bad_words_ids get_tokens_as_list word_list mess output_ids model generate inputs input_ids max_new_tokens 5 bad_words_ids bad_words_ids pad_token_id tokenizer eos_token_id print tokenizer batch_decode output_ids skip_special_tokens true 0 in a word the cake is a bit of a surprise filter eos token from bad_words_ids forbidding a sequence is equivalent to setting its bias to inf logitsprocessor that enforces constrained generation and is useful for prefix conditioned constrained generation see autoregressive entity retrieval https arxiv org abs 2010 00904 for more information args prefix_allowed_tokens_fn callable int torch tensor list int this function constraints the beam search to allowed tokens only at each step this function takes 2 arguments inputs_ids and the batch id batch_id it has to return a list with the allowed tokens for the next generation step conditioned on the previously generated tokens inputs_ids and the batch id batch_id logitsprocessor that enforces diverse beam search note that this logits processor is only effective for pretrainedmodel group_beam_search see diverse beam search decoding diverse solutions from neural sequence models https arxiv org pdf 1610 02424 pdf for more details tip diverse beam search can be particularly useful in scenarios where a variety of different outputs is desired rather than multiple similar sequences it allows the model to explore different generation paths and provides a broader coverage of possible outputs tip tip warning true this logits processor can be resource intensive especially when using large models or long sequences tip traditional beam search often generates very similar sequences across different beams hammingdiversitylogitsprocessor addresses this by penalizing beams that generate tokens already chosen by other beams in the same time step how it works grouping beams beams are divided into groups each group selects tokens independently of the others penalizing repeated tokens if a beam in a group selects a token already chosen by another group in the same step a penalty is applied to that token s score promoting diversity this penalty discourages beams within a group from selecting the same tokens as beams in other groups benefits diverse outputs produces a variety of different sequences exploration allows the model to explore different paths args diversity_penalty float this value is subtracted from a beam s score if it generates a token same as any beam from other group at a particular time note that diversity_penalty is only effective if group beam search is enabled the penalty applied to a beam s score when it generates a token that has already been chosen by another beam within the same group during the same time step a higher diversity_penalty will enforce greater diversity among the beams making it less likely for multiple beams to choose the same token conversely a lower penalty will allow beams to more freely choose similar tokens adjusting this value can help strike a balance between diversity and natural likelihood num_beams int number of beams used for group beam search beam search is a method used that maintains beams or multiple hypotheses at each step expanding each one and keeping the top scoring sequences a higher num_beams will explore more potential sequences this can increase chances of finding a high quality output but also increases computational cost num_beam_groups int number of groups to divide num_beams into in order to ensure diversity among different groups of beams each group of beams will operate independently selecting tokens without considering the choices of other groups this division promotes diversity by ensuring that beams within different groups explore different paths for instance if num_beams is 6 and num_beam_groups is 2 there will be 2 groups each containing 3 beams the choice of num_beam_groups should be made considering the desired level of output diversity and the total number of beams see this paper https arxiv org pdf 1610 02424 pdf for more details examples python from transformers import autotokenizer automodelforseq2seqlm import torch initialize the model and tokenizer tokenizer autotokenizer from_pretrained t5 base model automodelforseq2seqlm from_pretrained t5 base a long text about the solar system text the solar system is a gravitationally bound system comprising the sun and the objects that orbit it either directly or indirectly of the objects that orbit the sun directly the largest are the eight planets with the remainder being smaller objects such as the five dwarf planets and small solar system bodies the solar system formed 4 6 billion years ago from the gravitational collapse of a giant interstellar molecular cloud inputs tokenizer summarize text return_tensors pt generate diverse summary outputs_diverse model generate inputs num_beam_groups 2 diversity_penalty 10 0 max_length 100 num_beams 4 num_return_sequences 2 summaries_diverse tokenizer batch_decode outputs_diverse skip_special_tokens true generate non diverse summary outputs_non_diverse model generate inputs max_length 100 num_beams 4 num_return_sequences 2 summary_non_diverse tokenizer batch_decode outputs_non_diverse skip_special_tokens true with diversity_penalty the resulting beams are much more diverse print summary_non_diverse the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets print summaries_diverse the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the solar system formed 4 6 billion years ago from the collapse of a giant interstellar molecular cloud of the objects that orbit the sun directly the largest are the eight planets the rest of the objects are smaller objects such as the five dwarf planets and small solar system bodies args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary what are input ids glossary input ids scores torch floattensor of shape batch_size config vocab_size prediction scores of a language modeling head these can be logits for each vocabulary when not using beam search or log softmax for each vocabulary token when using beam search current_tokens torch longtensor of shape batch_size indices of input sequence tokens in the vocabulary corresponding to the tokens selected by the other beam groups in the current generation step beam_group_idx int the index of the beam group currently being processed return torch floattensor of shape batch_size config vocab_size the processed prediction scores hamming diversity penalise using same token in current group which was used in previous groups at the same time step predicted tokens of last time step of previous groups logitsprocessor that enforces the specified token as the first generated token args bos_token_id int the id of the token to force as the first generated token logitsprocessor that enforces the specified token as the last generated token when max_length is reached args max_length int the maximum length of the sequence to be generated eos_token_id union int list int the id of the token to force as the last generated token when max_length is reached optionally use a list to set multiple end of sequence tokens logitsprocessor that removes all nan and inf values to avoid the generation method to fail note that using the logits processor should only be used if necessary since it can slow down the generation method set all nan values to 0 0 set all inf values to max min possible value logitsprocessor that exponentially increases the score of the eos_token_id after start_index has been reached this allows generating shorter sequences without having a hard cutoff allowing the eos_token to be predicted in a meaningful position args exponential_decay_length_penalty tuple int float this tuple shall consist of start_index decay_factor where start_index indicates where penalty starts and decay_factor represents the factor of exponential decay eos_token_id union int list int the id of the end of sequence token optionally use a list to set multiple end of sequence tokens input_ids_seq_length int the length of the input sequence examples python from transformers import autotokenizer automodelforcausallm set_seed model automodelforcausallm from_pretrained gpt2 tokenizer autotokenizer from_pretrained gpt2 text just wanted to let you know i inputs tokenizer text return_tensors pt let s consider that we want short sentences so we limit max_length 30 however we observe that the answer tends to end abruptly set_seed 1 outputs model generate inputs do_sample true temperature 0 9 max_length 30 pad_token_id 50256 print tokenizer batch_decode outputs 0 just wanted to let you know i received a link to an ebook the book how to start a social network which was published in 2010 although to promote the appearance of the eos token at the right time we add the exponential_decay_length_penalty start_index decay_factor instead of cutting at max_tokens the output comes to an end before and usually with more meaning what happens is that starting from start_index the eos token score will be increased by decay_factor exponentially however if you set a high decay factor you may also end up with abruptly ending sequences set_seed 1 outputs model generate inputs do_sample true temperature 0 9 max_length 30 pad_token_id 50256 exponential_decay_length_penalty 15 1 6 print tokenizer batch_decode outputs 0 just wanted to let you know i received a link to an ebook the book how to start a social network which endoftext with a small decay factor you will have a higher chance of getting a meaningful sequence set_seed 1 outputs model generate inputs do_sample true temperature 0 9 max_length 30 pad_token_id 50256 exponential_decay_length_penalty 15 1 01 print tokenizer batch_decode outputs 0 just wanted to let you know i received a link to an ebook the book how to start a social network which was published in 2010 endoftext to support negative logits we compute the penalty of the absolute value and add to the original logit logitswarper and logitsprocessor for normalizing the scores using log softmax it s important to normalize the scores during beam search after applying the logits processors or warpers since the search algorithm used in this library doesn t do it it only does it before but they may need re normalization but it still supposes that the scores are normalized when comparing the hypotheses suppresstokensatbeginlogitsprocessor supresses a list of tokens as soon as the generate function starts generating using begin_index tokens this should ensure that the tokens defined by begin_suppress_tokens at not sampled at the begining of the generation this processor can be used to suppress a list of tokens the processor will set their log probs to inf so that they are not sampled this processor takes a list of pairs of integers which indicates a mapping from generation indices to token indices that will be forced before sampling the processor will set their log probs to inf so that they are sampled at their corresponding index logitsprocessor that modifies the logits for the generation of timestamps in the transcription when the input tokens are at a specific threshold the processor sets the scores to negative infinity the processor makes sure that timestamp tokens appear in pairs by masking out the logits that would break this pairing pattern this is done to maintain the consistency and structure of generated timestamps it also ensures that when the predicted probability of sampling any of the timestamp token is greater than any individual non timestamp token those non timestamp logits are set to negative infinity this is done to ensure the generation of timestamps over other potential tokens see the paper https arxiv org abs 2212 04356 for more information args generate_config generateconfig the generate config used to generate the output the following parameters are required eos_token_id int optional defaults to 50257 the id of the end of sequence token no_timestamps_token_id int optional defaults to 50363 the id of the notimestamps token max_initial_timestamp_index int optional defaults to 1 used to set the maximum value of the initial timestamp this is used to prevent the model from predicting timestamps that are too far in the future _detect_timestamp_from_logprob bool optional whether timestamps can be predicted from logprobs over all timestamps examples python import torch from transformers import autoprocessor whisperforconditionalgeneration generationconfig from datasets import load_dataset processor autoprocessor from_pretrained openai whisper tiny en model whisperforconditionalgeneration from_pretrained openai whisper tiny en ds load_dataset hf internal testing librispeech_asr_dummy clean split validation inputs processor ds 3 audio array return_tensors pt input_features inputs input_features displaying timestamps generated_ids model generate inputs input_features return_timestamps true transcription processor batch_decode generated_ids skip_special_tokens false 0 print transcription transcription transcription startoftranscript 0 00 he has grave doubts whether sir frederick layton s work is really greek after all and can 6 44 6 44 discover in it but little of rocky ithaca 9 44 endoftext no timestamps change eos this allows the user to select a specific token to terminate the sequence on in this case it s the word can 460 model generation_config eos_token_id 460 generated_ids model generate inputs input_features return_timestamps false transcription processor batch_decode generated_ids skip_special_tokens true 0 print transcription transcription transcription he has grave doubts whether sir frederick layton s work is really greek after all and can support for the kwargs this variable is mostly just used for testing suppress notimestamps which is handled by without_timestamps timestamps have to appear in pairs except directly before eos_token mask logits accordingly has to be non timestamp cannot be normal text tokens timestamps shouldn t decrease forbid timestamp tokens smaller than the last the following lines of code are copied from https github com openai whisper pull 914 files r1137085090 avoid to emit 0 00 again apply the max_initial_timestamp option if sum of probability over timestamps is above any other token sample timestamp logits processor for classifier free guidance cfg the scores are split over the batch dimension where the first half correspond to the conditional logits predicted from the input prompt and the second half correspond to the unconditional logits predicted from an empty or null prompt the processor computes a weighted average across the conditional and unconditional logits parameterised by the guidance_scale see the paper https arxiv org abs 2306 05284 for more information args guidance_scale float the guidance scale for classifier free guidance cfg cfg is enabled by setting guidance_scale 1 higher guidance scale encourages the model to generate samples that are more closely linked to the input prompt usually at the expense of poorer quality simple check to make sure we have compatible batch sizes between our logits scores cond uncond and input ids cond only logitsprocessor enforcing alternated generation between the two codebooks of bark s fine submodel args input_start_len int the length of the initial input sequence semantic_vocab_size int vocabulary size of the semantic part i e number of tokens associated to the semantic vocabulary codebook_size int number of tokens associated to the codebook even first codebook odd second codebook logits processor for classifier free guidance cfg the processors computes a weighted average across scores from prompt conditional and prompt unconditional or negative logits parameterized by the guidance_scale the unconditional scores are computed internally by prompting model with the unconditional_ids branch see the paper https arxiv org abs 2306 17806 for more information args guidance_scale float the guidance scale for classifier free guidance cfg cfg is enabled by setting guidance_scale 1 higher guidance scale encourages the model to generate samples that are more closely linked to the input prompt usually at the expense of poorer quality a value smaller than 1 has the opposite effect while making the negative prompt provided with negative_prompt_ids if any act as a positive prompt model pretrainedmodel the model computing the unconditional scores supposedly the same as the one computing the conditional scores both models must use the same tokenizer unconditional_ids torch longtensor of shape batch_size sequence_length optional indices of input sequence tokens in the vocabulary for the unconditional branch if unset will default to the last token of the prompt unconditional_attention_mask torch longtensor of shape batch_size sequence_length optional attention mask for unconditional_ids use_cache bool optional defaults to true whether to cache key values during the negative prompt forward pass examples python from transformers import autotokenizer automodelforcausallm model automodelforcausallm from_pretrained gpt2 tokenizer autotokenizer from_pretrained gpt2 inputs tokenizer today a dragon flew over paris france return_tensors pt out model generate inputs input_ids guidance_scale 1 5 tokenizer batch_decode out skip_special_tokens true 0 today a dragon flew over paris france killing at least 50 people and injuring more than 100 with a negative prompt neg_inputs tokenizer a very happy event happened return_tensors pt out model generate inputs input_ids guidance_scale 2 negative_prompt_ids neg_inputs input_ids tokenizer batch_decode out skip_special_tokens true 0 today a dragon flew over paris france killing at least 130 people french media reported that with a positive prompt neg_inputs tokenizer a very happy event happened return_tensors pt out model generate inputs input_ids guidance_scale 0 negative_prompt_ids neg_inputs input_ids tokenizer batch_decode out skip_special_tokens true 0 today a dragon flew over paris france and i m very happy to be here i this processor ensures that the eos token is selected if its probability is greater than the min_eos_p args eos_token_id union int list int the id of the end of sequence token optionally use a list to set multiple end of sequence tokens min_eos_p float optional minimum end of speech threshold create scores full of inf except for the eos_token_id
import inspect
import math
from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union

import numpy as np
import torch

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


logger = get_logger(__name__)


LOGITS_PROCESSOR_INPUTS_DOCSTRING = r


class LogitsProcessor:
    

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        raise NotImplementedError(
            f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."
        )


class LogitsWarper:
    

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        raise NotImplementedError(
            f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."
        )


class LogitsProcessorList(list):
    

    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
        r
        for processor in self:
            function_args = inspect.signature(processor.__call__).parameters
            if len(function_args) > 2:
                if not all(arg in kwargs for arg in list(function_args.keys())[2:]):
                    raise ValueError(
                        f"Make sure that all the required parameters: {list(function_args.keys())} for "
                        f"{processor.__class__} are passed to the logits processor."
                    )
                scores = processor(input_ids, scores, **kwargs)
            else:
                scores = processor(input_ids, scores)
        return scores


class MinLengthLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, min_length: int, eos_token_id: Union[int, List[int]]):
        if not isinstance(min_length, int) or min_length < 0:
            raise ValueError(f"`min_length` has to be a non-negative integer, but is {min_length}")

        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        if not all(isinstance(i, int) for i in eos_token_id) or any(i < 0 for i in eos_token_id):
            logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}")

        self.min_length = min_length
        self.eos_token_id = eos_token_id

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if cur_len < self.min_length:
            for i in self.eos_token_id:
                scores[:, i] = -float("inf")
        return scores


class MinNewTokensLengthLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, prompt_length_to_skip: int, min_new_tokens: int, eos_token_id: Union[int, List[int]]):
        for arg_name, arg_value in [
            ("prompt_length_to_skip", prompt_length_to_skip),
            ("min_new_tokens", min_new_tokens),
        ]:
            if not isinstance(arg_value, int) or arg_value < 0:
                raise ValueError(f"`{arg_name}` has to be a positive integer, but is {arg_value}")

        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        if not all(isinstance(i, int) for i in eos_token_id) or any(i < 0 for i in eos_token_id):
            logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}")

        self.prompt_length_to_skip = prompt_length_to_skip
        self.min_new_tokens = min_new_tokens
        self.eos_token_id = eos_token_id

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        new_tokens_length = input_ids.shape[-1] - self.prompt_length_to_skip
        if new_tokens_length < self.min_new_tokens:
            for i in self.eos_token_id:
                scores[:, i] = -float("inf")

        return scores


class TemperatureLogitsWarper(LogitsWarper):
    r

    def __init__(self, temperature: float):
        if not isinstance(temperature, float) or not (temperature > 0):
            except_msg = (
                f"`temperature` (={temperature}) has to be a strictly positive float, otherwise your next token "
                "scores will be invalid."
            )
            if isinstance(temperature, float) and temperature == 0.0:
                except_msg += " If you're looking for greedy decoding strategies, set `do_sample=False`."
            raise ValueError(except_msg)

        self.temperature = temperature

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        scores = scores / self.temperature
        return scores


class RepetitionPenaltyLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, penalty: float):
        if not isinstance(penalty, float) or not (penalty > 0):
            raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")

        self.penalty = penalty

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        score = torch.gather(scores, 1, input_ids)

        
        score = torch.where(score < 0, score * self.penalty, score / self.penalty)

        scores.scatter_(1, input_ids, score)
        return scores


class EncoderRepetitionPenaltyLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, penalty: float, encoder_input_ids: torch.LongTensor):
        if not isinstance(penalty, float) or not (penalty > 0):
            raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}")

        self.penalty = 1 / penalty
        self.encoder_input_ids = encoder_input_ids

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        score = torch.gather(scores, 1, self.encoder_input_ids)

        
        score = torch.where(score < 0, score * self.penalty, score / self.penalty)

        scores.scatter_(1, self.encoder_input_ids, score)
        return scores


class TopPLogitsWarper(LogitsWarper):
    

    def __init__(self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
        top_p = float(top_p)
        if top_p < 0 or top_p > 1.0:
            raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}")
        if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1):
            raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}")

        self.top_p = top_p
        self.filter_value = filter_value
        self.min_tokens_to_keep = min_tokens_to_keep

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        sorted_logits, sorted_indices = torch.sort(scores, descending=False)
        cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)

        
        sorted_indices_to_remove = cumulative_probs <= (1 - self.top_p)
        
        sorted_indices_to_remove[..., -self.min_tokens_to_keep :] = 0

        
        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
        scores = scores.masked_fill(indices_to_remove, self.filter_value)
        return scores


class TopKLogitsWarper(LogitsWarper):
    r

    def __init__(self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
        if not isinstance(top_k, int) or top_k <= 0:
            raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}")

        self.top_k = max(top_k, min_tokens_to_keep)
        self.filter_value = filter_value

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        top_k = min(self.top_k, scores.size(-1))  
        
        indices_to_remove = scores < torch.topk(scores, top_k)[0][..., -1, None]
        scores = scores.masked_fill(indices_to_remove, self.filter_value)
        return scores


class TypicalLogitsWarper(LogitsWarper):
    r

    def __init__(self, mass: float = 0.9, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
        mass = float(mass)
        if not (mass > 0 and mass < 1):
            raise ValueError(f"`typical_p` has to be a float > 0 and < 1, but is {mass}")
        if not isinstance(min_tokens_to_keep, int) or (min_tokens_to_keep < 1):
            raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}")

        self.filter_value = filter_value
        self.mass = mass
        self.min_tokens_to_keep = min_tokens_to_keep

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        normalized = torch.nn.functional.log_softmax(scores, dim=-1)
        p = torch.exp(normalized)
        ent = -(normalized * p).nansum(-1, keepdim=True)

        
        shifted_scores = torch.abs((-normalized) - ent)
        sorted_scores, sorted_indices = torch.sort(shifted_scores, descending=False)
        sorted_logits = scores.gather(-1, sorted_indices)
        cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)

        
        last_ind = (cumulative_probs < self.mass).sum(dim=1)
        last_ind.clamp_(max=sorted_scores.shape[-1] - 1)
        sorted_indices_to_remove = sorted_scores > sorted_scores.gather(1, last_ind.view(-1, 1))
        sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0
        indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)

        scores = scores.masked_fill(indices_to_remove, self.filter_value)
        return scores


class EpsilonLogitsWarper(LogitsWarper):
    r

    def __init__(self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
        epsilon = float(epsilon)
        if epsilon <= 0 or epsilon >= 1:
            raise ValueError(f"`epsilon_cutoff` has to be a float > 0 and < 1, but is {epsilon}")

        min_tokens_to_keep = int(min_tokens_to_keep)
        if min_tokens_to_keep < 1:
            raise ValueError(
                f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}"
            )

        self.epsilon = epsilon
        self.filter_value = filter_value
        self.min_tokens_to_keep = min_tokens_to_keep

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        probabilities = scores.softmax(dim=-1)
        indices_to_remove = probabilities < self.epsilon

        
        top_k = min(self.min_tokens_to_keep, scores.size(-1))  
        indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None])

        scores = scores.masked_fill(indices_to_remove, self.filter_value)
        return scores


class EtaLogitsWarper(LogitsWarper):
    r

    def __init__(self, epsilon: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1):
        epsilon = float(epsilon)
        if epsilon <= 0 or epsilon >= 1:
            raise ValueError(f"`eta_cutoff` has to be a float > 0 and < 1, but is {epsilon}")

        min_tokens_to_keep = int(min_tokens_to_keep)
        if min_tokens_to_keep < 1:
            raise ValueError(
                f"`min_tokens_to_keep` has to be a strictly positive integer, but is {min_tokens_to_keep}"
            )

        self.epsilon = torch.tensor(epsilon)
        self.filter_value = filter_value
        self.min_tokens_to_keep = min_tokens_to_keep

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        probabilities = scores.softmax(dim=-1)
        entropy = torch.distributions.Categorical(logits=scores).entropy()
        eta = torch.min(self.epsilon, torch.sqrt(self.epsilon) * torch.exp(-entropy))[..., None]
        indices_to_remove = probabilities < eta

        
        top_k = min(self.min_tokens_to_keep, scores.size(-1))  
        indices_to_remove = indices_to_remove & (scores < torch.topk(scores, top_k)[0][..., -1, None])

        scores = scores.masked_fill(indices_to_remove, self.filter_value)
        return scores


def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int):
    
    
    generated_ngrams = [{} for _ in range(num_hypos)]
    for idx in range(num_hypos):
        gen_tokens = prev_input_ids[idx].tolist()
        generated_ngram = generated_ngrams[idx]
        
        for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]):
            prev_ngram_tuple = tuple(ngram[:-1])
            generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ngram[-1]]
    return generated_ngrams


def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len):
    
    
    start_idx = cur_len + 1 - ngram_size
    ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist())
    return banned_ngrams.get(ngram_idx, [])


def _calc_banned_ngram_tokens(
    ngram_size: int, prev_input_ids: torch.Tensor, num_hypos: int, cur_len: int
) -> List[Iterable[int]]:
    
    if cur_len + 1 < ngram_size:
        
        return [[] for _ in range(num_hypos)]
    generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos)
    banned_tokens = [
        _get_generated_ngrams(generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len)
        for hypo_idx in range(num_hypos)
    ]
    return banned_tokens


class NoRepeatNGramLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, ngram_size: int):
        if not isinstance(ngram_size, int) or ngram_size <= 0:
            raise ValueError(f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}")
        self.ngram_size = ngram_size

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        num_batch_hypotheses = scores.shape[0]
        cur_len = input_ids.shape[-1]
        banned_batch_tokens = _calc_banned_ngram_tokens(self.ngram_size, input_ids, num_batch_hypotheses, cur_len)
        for i, banned_tokens in enumerate(banned_batch_tokens):
            scores[i, banned_tokens] = -float("inf")

        return scores


class EncoderNoRepeatNGramLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, encoder_ngram_size: int, encoder_input_ids: torch.LongTensor):
        if not isinstance(encoder_ngram_size, int) or encoder_ngram_size <= 0:
            raise ValueError(
                f"`encoder_ngram_size` has to be a strictly positive integer, but is {encoder_ngram_size}"
            )
        self.ngram_size = encoder_ngram_size
        if len(encoder_input_ids.shape) == 1:
            encoder_input_ids = encoder_input_ids.unsqueeze(0)
        self.batch_size = encoder_input_ids.shape[0]
        self.generated_ngrams = _get_ngrams(encoder_ngram_size, encoder_input_ids, self.batch_size)

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        num_hypos = scores.shape[0]
        num_beams = num_hypos // self.batch_size
        cur_len = input_ids.shape[-1]
        banned_batch_tokens = [
            _get_generated_ngrams(
                self.generated_ngrams[hypo_idx // num_beams], input_ids[hypo_idx], self.ngram_size, cur_len
            )
            for hypo_idx in range(num_hypos)
        ]

        for i, banned_tokens in enumerate(banned_batch_tokens):
            scores[i, banned_tokens] = -float("inf")

        return scores


class SequenceBiasLogitsProcessor(LogitsProcessor):
    

    def __init__(self, sequence_bias: Dict[Tuple[int], float]):
        self.sequence_bias = sequence_bias
        self._validate_arguments()

        
        
        self.length_1_bias = None
        self.prepared_bias_variables = False

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        if not self.prepared_bias_variables:
            self._prepare_bias_variables(scores)

        
        bias = torch.zeros_like(scores)

        
        bias += self.length_1_bias

        
        for sequence_ids, sequence_bias in self.sequence_bias.items():
            if len(sequence_ids) == 1:  
                continue
            if len(sequence_ids) > input_ids.shape[1]:  
                continue
            prefix_length = len(sequence_ids) - 1
            last_token = sequence_ids[-1]
            matching_rows = torch.eq(
                input_ids[:, -prefix_length:],
                torch.tensor(sequence_ids[:-1], dtype=input_ids.dtype, device=input_ids.device),
            ).prod(dim=1)
            bias[:, last_token] += torch.where(
                matching_rows.bool(),
                torch.tensor(sequence_bias, device=input_ids.device),
                torch.tensor(0.0, device=input_ids.device),
            )

        
        scores = scores + bias
        return scores

    def _prepare_bias_variables(self, scores: torch.FloatTensor):
        vocabulary_size = scores.shape[-1]

        
        invalid_biases = []
        for sequence_ids in self.sequence_bias:
            for token_id in sequence_ids:
                if token_id >= vocabulary_size:
                    invalid_biases.append(token_id)
        if len(invalid_biases) > 0:
            raise ValueError(
                f"The model vocabulary size is {vocabulary_size}, but the following tokens were being biased: "
                f"{invalid_biases}"
            )

        
        
        self.length_1_bias = torch.zeros((vocabulary_size,), dtype=torch.float).to(scores.device)
        for sequence_ids, bias in self.sequence_bias.items():
            if len(sequence_ids) == 1:
                self.length_1_bias[sequence_ids[-1]] = bias

        self.prepared_bias_variables = True

    def _validate_arguments(self):
        sequence_bias = self.sequence_bias
        if not isinstance(sequence_bias, dict) or len(sequence_bias) == 0:
            raise ValueError(f"`sequence_bias` has to be a non-empty dictionary, but is {sequence_bias}.")
        if any(not isinstance(sequence_ids, tuple) for sequence_ids in sequence_bias.keys()):
            raise ValueError(f"`sequence_bias` has to be a dict with tuples as keys, but is {sequence_bias}.")
        if any(
            any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in sequence_ids)
            or len(sequence_ids) == 0
            for sequence_ids in sequence_bias.keys()
        ):
            raise ValueError(
                f"Each key in `sequence_bias` has to be a non-empty tuple of positive integers, but is "
                f"{sequence_bias}."
            )
        if any(not isinstance(bias, float) for bias in sequence_bias.values()):
            raise ValueError(f"`sequence_bias` has to be a dict with floats as values, but is {sequence_bias}.")


class NoBadWordsLogitsProcessor(SequenceBiasLogitsProcessor):
    

    def __init__(self, bad_words_ids: List[List[int]], eos_token_id: Union[int, List[int]]):
        self.bad_word_ids = bad_words_ids
        self._validate_arguments()

        
        if eos_token_id is None:
            eos_token_id = []
        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        bad_words_ids = list(
            filter(lambda bad_token_seq: all(bad_token_seq != [i] for i in eos_token_id), bad_words_ids)
        )

        
        sequence_bias = {tuple(sequence): float("-inf") for sequence in bad_words_ids}
        super().__init__(sequence_bias=sequence_bias)

    def _validate_arguments(self):
        bad_words_ids = self.bad_word_ids
        if not isinstance(bad_words_ids, list) or len(bad_words_ids) == 0:
            raise ValueError(f"`bad_words_ids` has to be a non-empty list, but is {bad_words_ids}.")
        if any(not isinstance(bad_word_ids, list) for bad_word_ids in bad_words_ids):
            raise ValueError(f"`bad_words_ids` has to be a list of lists, but is {bad_words_ids}.")
        if any(
            any((not isinstance(token_id, (int, np.integer)) or token_id < 0) for token_id in bad_word_ids)
            for bad_word_ids in bad_words_ids
        ):
            raise ValueError(
                f"Each list in `bad_words_ids` has to be a list of positive integers, but is {bad_words_ids}."
            )


class PrefixConstrainedLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], List[int]], num_beams: int):
        self._prefix_allowed_tokens_fn = prefix_allowed_tokens_fn
        self._num_beams = num_beams

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        mask = torch.full_like(scores, -math.inf)
        for batch_id, beam_sent in enumerate(input_ids.view(-1, self._num_beams, input_ids.shape[-1])):
            for beam_id, sent in enumerate(beam_sent):
                mask[batch_id * self._num_beams + beam_id, self._prefix_allowed_tokens_fn(batch_id, sent)] = 0

        return scores + mask


class HammingDiversityLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, diversity_penalty: float, num_beams: int, num_beam_groups: int):
        if not isinstance(diversity_penalty, float) or (not diversity_penalty > 0.0):
            raise ValueError("`diversity_penalty` should be a float strictly larger than 0.")
        self._diversity_penalty = diversity_penalty
        if not isinstance(num_beams, int) or num_beams < 2:
            raise ValueError("`num_beams` should be an integer strictly larger than 1.")
        self._num_beams = num_beams
        if not isinstance(num_beam_groups, int) or num_beam_groups < 2:
            raise ValueError("`num_beam_groups` should be an integer strictly larger than 1.")
        if num_beam_groups > num_beams:
            raise ValueError("`beam_groups` has to be smaller or equal to `num_beams`.")
        self._num_sub_beams = num_beams // num_beam_groups

    def __call__(
        self,
        input_ids: torch.LongTensor,
        scores: torch.FloatTensor,
        current_tokens: torch.LongTensor,
        beam_group_idx: int,
    ) -> torch.FloatTensor:
        r
        
        
        batch_size = current_tokens.shape[0] // self._num_beams
        group_start_idx = beam_group_idx * self._num_sub_beams
        group_end_idx = min(group_start_idx + self._num_sub_beams, self._num_beams)
        group_size = group_end_idx - group_start_idx
        vocab_size = scores.shape[-1]

        if group_start_idx == 0:
            return scores

        for batch_idx in range(batch_size):
            
            previous_group_tokens = current_tokens[
                batch_idx * self._num_beams : batch_idx * self._num_beams + group_start_idx
            ]
            token_frequency = torch.bincount(previous_group_tokens, minlength=vocab_size).to(scores.device)
            scores[batch_idx * group_size : (batch_idx + 1) * group_size] -= self._diversity_penalty * token_frequency

        return scores


class ForcedBOSTokenLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, bos_token_id: int):
        self.bos_token_id = bos_token_id

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if cur_len == 1:
            num_tokens = scores.shape[1]
            scores[:, [i for i in range(num_tokens) if i != self.bos_token_id]] = -float("inf")
            scores[:, self.bos_token_id] = 0
        return scores


class ForcedEOSTokenLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, max_length: int, eos_token_id: Union[int, List[int]]):
        self.max_length = max_length
        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        self.eos_token_id = eos_token_id

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if cur_len == self.max_length - 1:
            num_tokens = scores.shape[1]
            scores[:, [i for i in range(num_tokens) if i not in self.eos_token_id]] = -float("inf")
            for i in self.eos_token_id:
                scores[:, i] = 0
        return scores


class InfNanRemoveLogitsProcessor(LogitsProcessor):
    r

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        scores[scores != scores] = 0.0

        
        scores[scores == float("inf")] = torch.finfo(scores.dtype).max
        scores[scores == float("-inf")] = torch.finfo(scores.dtype).min

        return scores


class ExponentialDecayLengthPenalty(LogitsProcessor):
    r

    def __init__(
        self,
        exponential_decay_length_penalty: Tuple[int, float],
        eos_token_id: Union[int, List[int]],
        input_ids_seq_length: int,
    ):
        self.regulation_start = exponential_decay_length_penalty[0] + input_ids_seq_length
        self.regulation_factor = exponential_decay_length_penalty[1]
        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        self.eos_token_id = eos_token_id

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        cur_len = input_ids.shape[-1]
        if cur_len > self.regulation_start:
            for i in self.eos_token_id:
                penalty_idx = cur_len - self.regulation_start
                
                scores[:, i] = scores[:, i] + torch.abs(scores[:, i]) * (pow(self.regulation_factor, penalty_idx) - 1)
        return scores


class LogitNormalization(LogitsProcessor, LogitsWarper):
    r

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        scores = scores.log_softmax(dim=-1)
        return scores


class SuppressTokensAtBeginLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, begin_suppress_tokens, begin_index):
        self.begin_suppress_tokens = list(begin_suppress_tokens)
        self.begin_index = begin_index

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        if input_ids.shape[1] == self.begin_index:
            scores[:, self.begin_suppress_tokens] = -float("inf")

        return scores


class SuppressTokensLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, suppress_tokens):
        self.suppress_tokens = list(suppress_tokens)

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        scores[:, self.suppress_tokens] = -float("inf")
        return scores


class ForceTokensLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, force_token_map: List[List[int]]):
        self.force_token_map = dict(force_token_map)

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        generation_idx = input_ids.shape[-1]
        current_token = self.force_token_map.get(generation_idx, None)
        if current_token is not None:
            scores[:, :] = -float("inf")
            scores[:, current_token] = 0
        return scores


class WhisperTimeStampLogitsProcessor(LogitsProcessor):
    r

    def __init__(
        self, generate_config, _detect_timestamp_from_logprob: Optional[bool] = None
    ):  
        self.eos_token_id = generate_config.eos_token_id
        self.no_timestamps_token_id = generate_config.no_timestamps_token_id
        self.timestamp_begin = generate_config.no_timestamps_token_id + 1

        
        self._detect_timestamp_from_logprob = (
            _detect_timestamp_from_logprob
            if _detect_timestamp_from_logprob is not None
            else getattr(generate_config, "_detect_timestamp_from_logprob", True)
        )

        self.begin_index = (
            len(generate_config.forced_decoder_ids) + 1 if generate_config.forced_decoder_ids is not None else 1
        )
        self.max_initial_timestamp_index = getattr(generate_config, "max_initial_timestamp_index", None)

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        scores[:, self.no_timestamps_token_id] = -float("inf")

        
        for k in range(input_ids.shape[0]):
            sampled_tokens = input_ids[k, self.begin_index :]
            seq = list(sampled_tokens.tolist())

            last_was_timestamp = len(seq) >= 1 and seq[-1] >= self.timestamp_begin
            penultimate_was_timestamp = len(seq) < 2 or seq[-2] >= self.timestamp_begin

            if last_was_timestamp:
                if penultimate_was_timestamp:  
                    scores[k, self.timestamp_begin :] = -float("inf")
                else:  
                    scores[k, : self.eos_token_id] = -float("inf")

            timestamps = sampled_tokens[sampled_tokens.ge(self.timestamp_begin)]
            if timestamps.numel() > 0:
                
                
                if last_was_timestamp and not penultimate_was_timestamp:
                    timestamp_last = timestamps[-1]
                else:
                    
                    timestamp_last = timestamps[-1] + 1

                scores[k, self.timestamp_begin : timestamp_last] = -float("inf")

        
        if input_ids.shape[1] == self.begin_index:
            scores[:, : self.timestamp_begin] = -float("inf")

            if self.max_initial_timestamp_index is not None:
                last_allowed = self.timestamp_begin + self.max_initial_timestamp_index
                scores[:, last_allowed + 1 :] = -float("inf")

        
        logprobs = torch.nn.functional.log_softmax(scores.float(), dim=-1)
        for k in range(input_ids.shape[0]):
            timestamp_logprob = logprobs[k, self.timestamp_begin :].logsumexp(dim=-1)
            max_text_token_logprob = logprobs[k, : self.timestamp_begin].max()
            if timestamp_logprob > max_text_token_logprob and self._detect_timestamp_from_logprob:
                scores[k, : self.timestamp_begin] = -float("inf")

        return scores


class ClassifierFreeGuidanceLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, guidance_scale):
        if guidance_scale > 1:
            self.guidance_scale = guidance_scale
        else:
            raise ValueError(
                "Require guidance scale >1 to use the classifier free guidance processor, got guidance scale "
                f"{guidance_scale}."
            )

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        
        
        if scores.shape[0] != 2 * input_ids.shape[0]:
            raise ValueError(
                f"Logits should have twice the batch size of the input ids, the first half of batches corresponding to "
                f"the conditional inputs, and the second half of batches corresponding to the unconditional inputs. Got "
                f"batch size {scores.shape[0]} for the logits and {input_ids.shape[0]} for the input ids."
            )
        unguided_bsz = scores.shape[0] // 2
        cond_logits, uncond_logits = scores.split(unguided_bsz, dim=0)
        scores = uncond_logits + (cond_logits - uncond_logits) * self.guidance_scale
        return scores


class AlternatingCodebooksLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, input_start_len: int, semantic_vocab_size: int, codebook_size: int):
        if not isinstance(input_start_len, int) or input_start_len < 0:
            raise ValueError(f"`input_starting_length` has to be a non-negative integer, but is {input_start_len}")

        self.input_start_len = input_start_len
        self.semantic_vocab_size = semantic_vocab_size
        self.codebook_size = codebook_size

    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        curr_len = input_ids.shape[-1]

        
        is_first_codebook = ((curr_len - self.input_start_len) % 2) == 0

        if is_first_codebook:
            scores[:, : self.semantic_vocab_size] = -float("inf")
            scores[:, self.semantic_vocab_size + self.codebook_size :] = -float("inf")
        else:
            scores[:, : self.semantic_vocab_size + self.codebook_size] = -float("inf")

        return scores


class UnbatchedClassifierFreeGuidanceLogitsProcessor(LogitsProcessor):
    r

    def __init__(
        self,
        guidance_scale: float,
        model,
        unconditional_ids: Optional[torch.LongTensor] = None,
        unconditional_attention_mask: Optional[torch.LongTensor] = None,
        use_cache: Optional[bool] = True,
    ):
        self.guidance_scale = guidance_scale
        self.model = model
        self.unconditional_context = {
            "input_ids": unconditional_ids,
            "attention_mask": unconditional_attention_mask,
            "use_cache": use_cache,
            "past_key_values": None,
            "first_pass": True,
        }

    def get_unconditional_logits(self, input_ids):
        if self.unconditional_context["first_pass"]:
            if self.unconditional_context["input_ids"] is None:
                self.unconditional_context["input_ids"] = input_ids[:, -1:]
            if self.unconditional_context["attention_mask"] is None:
                self.unconditional_context["attention_mask"] = torch.ones_like(
                    self.unconditional_context["input_ids"], dtype=torch.long
                )
            input_ids = self.unconditional_context["input_ids"]
            attention_mask = self.unconditional_context["attention_mask"]
            self.unconditional_context["first_pass"] = False
        else:
            attention_mask = torch.cat(
                [
                    self.unconditional_context["attention_mask"],
                    torch.ones_like(input_ids[:, -1:], dtype=torch.long),
                ],
                dim=1,
            )
            if not self.unconditional_context["use_cache"]:
                input_ids = torch.cat([self.unconditional_context["input_ids"], input_ids[:, -1:]], dim=1)
            else:
                input_ids = input_ids[:, -1:]
            self.unconditional_context["input_ids"] = input_ids
            self.unconditional_context["attention_mask"] = attention_mask

        out = self.model(
            input_ids,
            attention_mask=attention_mask,
            use_cache=self.unconditional_context["use_cache"],
            past_key_values=self.unconditional_context["past_key_values"],
        )
        self.unconditional_context["past_key_values"] = out.get("past_key_values", None)

        return out.logits

    def __call__(self, input_ids, scores):
        scores = torch.nn.functional.log_softmax(scores, dim=-1)
        if self.guidance_scale == 1:
            return scores

        logits = self.get_unconditional_logits(input_ids)

        unconditional_logits = torch.nn.functional.log_softmax(logits[:, -1], dim=-1)
        out = self.guidance_scale * (scores - unconditional_logits) + unconditional_logits
        return out


class BarkEosPrioritizerLogitsProcessor(LogitsProcessor):
    r

    def __init__(self, eos_token_id: Union[int, List[int]], min_eos_p: float):
        if isinstance(eos_token_id, int):
            eos_token_id = [eos_token_id]
        self.eos_token_id = eos_token_id
        if min_eos_p is not None and min_eos_p <= 0:
            raise ValueError(f"`min_eos_p` has to be a positive float, but is {min_eos_p}")
        self.min_eos_p = min_eos_p

    @add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        if self.min_eos_p:
            probs = torch.nn.functional.softmax(scores.float(), dim=-1)
            
            early_stop_scores = torch.ones_like(scores) * -float("inf")
            early_stop_scores[:, self.eos_token_id] = scores[:, self.eos_token_id]

            do_early_stop = probs[:, self.eos_token_id] > self.min_eos_p
            scores = torch.where(do_early_stop, early_stop_scores, scores)

        return scores