codingutf8 2023 the pop2piano s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for pop2piano import json import os from typing import list optional tuple union import numpy as np from featureextractionutils import batchfeature from tokenizationutils import addedtoken batchencoding paddingstrategy pretrainedtokenizer truncationstrategy from utils import tensortype isprettymidiavailable logging requiresbackends tonumpy if isprettymidiavailable import prettymidi logger logging getloggername vocabfilesnames vocab vocab json pretrainedvocabfilesmap vocab sweetcocoapop2piano https huggingface cosweetcocoapop2pianoblobmainvocab json def tokentimetonotenumber cutofftimeidx currentidx currentidx number if cutofftimeidx is not none currentidx mincurrentidx cutofftimeidx return currentidx def tokennotetonotenumber currentvelocity defaultvelocity noteonsetsready currentidx notes if noteonsetsreadynumber is not none offset with onset onsetidx noteonsetsreadynumber if onsetidx currentidx time shift after previous noteon offsetidx currentidx notes appendonsetidx offsetidx number defaultvelocity onsetsready none if currentvelocity 0 else currentidx noteonsetsreadynumber onsetsready else noteonsetsreadynumber currentidx return notes class pop2pianotokenizerpretrainedtokenizer modelinputnames tokenids attentionmask vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap def init self vocab defaultvelocity77 numbars2 unktoken1 eostoken1 padtoken0 bostoken2 kwargs unktoken addedtokenunktoken lstripfalse rstripfalse if isinstanceunktoken str else unktoken eostoken addedtokeneostoken lstripfalse rstripfalse if isinstanceeostoken str else eostoken padtoken addedtokenpadtoken lstripfalse rstripfalse if isinstancepadtoken str else padtoken bostoken addedtokenbostoken lstripfalse rstripfalse if isinstancebostoken str else bostoken self defaultvelocity defaultvelocity self numbars numbars load the vocab with openvocab rb as file self encoder json loadfile create mappings for encoder self decoder v k for k v in self encoder items super init unktokenunktoken eostokeneostoken padtokenpadtoken bostokenbostoken kwargs property def vocabsizeself returns the vocabulary of the tokenizer return dictself encoder self addedtokensencoder def convertidtotokenself tokenid int list tokentypevalue self decoder gettokenid fself unktokentokentime tokentypevalue tokentypevalue split tokentype value jointokentypevalue1 inttokentypevalue0 return tokentype value def converttokentoidself token tokentypetokentime int return self encoder getftokentokentype intself unktoken def relativebatchtokensidstonotes self tokens np ndarray beatoffsetidx int barsperbatch int cutofftimeidx int notes none for index in rangelentokens tokens tokensindex startidx beatoffsetidx index barsperbatch 4 cutofftimeidx cutofftimeidx startidx notes self relativetokensidstonotes tokens startidxstartidx cutofftimeidxcutofftimeidx if lennotes 0 pass elif notes is none notes notes else notes np concatenatenotes notes axis0 if notes is none return return notes def relativebatchtokensidstomidi self tokens np ndarray beatstep np ndarray beatoffsetidx int 0 barsperbatch int 2 cutofftimeidx int 12 beatoffsetidx 0 if beatoffsetidx is none else beatoffsetidx notes self relativebatchtokensidstonotes tokenstokens beatoffsetidxbeatoffsetidx barsperbatchbarsperbatch cutofftimeidxcutofftimeidx midi self notestomidinotes beatstep offsetsecbeatstepbeatoffsetidx return midi taken from the original code please see https github comsweetcocoapop2pianoblobfac11e8dcfc73487513f4588e8d0c22a22f2fdc5miditokenizer pyl257 def relativetokensidstonotesself tokens np ndarray startidx float cutofftimeidx float none words self convertidtotokentoken for token in tokens currentidx startidx currentvelocity 0 noteonsetsready none for i in rangesumk endswithnote for k in self encoder keys 1 notes for tokentype number in words if tokentype tokenspecial if number 1 break elif tokentype tokentime currentidx tokentimetonote numbernumber cutofftimeidxcutofftimeidx currentidxcurrentidx elif tokentype tokenvelocity currentvelocity number elif tokentype tokennote notes tokennotetonote numbernumber currentvelocitycurrentvelocity defaultvelocityself defaultvelocity noteonsetsreadynoteonsetsready currentidxcurrentidx notesnotes else raise valueerrortoken type not understood for pitch noteonset in enumeratenoteonsetsready force offset if no offset for each pitch if noteonset is not none if cutofftimeidx is none cutoff noteonset 1 else cutoff maxcutofftimeidx noteonset 1 offsetidx maxcurrentidx cutoff notes appendnoteonset offsetidx pitch self defaultvelocity if lennotes 0 return else notes np arraynotes noteorder notes 0 128 notes 1 notes notesnoteorder argsort return notes def notestomidiself notes np ndarray beatstep np ndarray offsetsec int 0 0 requiresbackendsself prettymidi newpm prettymidi prettymidiresolution384 initialtempo120 0 newinst prettymidi instrumentprogram0 newnotes for onsetidx offsetidx pitch velocity in notes newnote prettymidi note velocityvelocity pitchpitch startbeatsteponsetidx offsetsec endbeatstepoffsetidx offsetsec newnotes appendnewnote newinst notes newnotes newpm instruments appendnewinst newpm removeinvalidnotes return newpm def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return save the encoder outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocab with openoutvocabfile w as file file writejson dumpsself encoder return outvocabfile def encodeplus self notes unionnp ndarray listprettymidi note truncationstrategy optionaltruncationstrategy none maxlength optionalint none kwargs batchencoding r this is the encodeplus method for pop2pianotokenizer it converts the midi notes to the transformer generated token ids it only works on a single batch to process multiple batches please use batchencodeplus or call method args notes numpy ndarray of shape sequencelength 4 or list of prettymidi note objects this represents the midi notes if notes is a numpy ndarray each sequence must have 4 values they are onset idx offset idx pitch and velocity if notes is a list containing prettymidi note objects each sequence must have 4 attributes they are start end pitch and velocity truncationstrategy tokenizationutilsbase truncationstrategy optional indicates the truncation strategy that is going to be used during truncation maxlength int optional maximum length of the returned list and optionally padding length see above returns batchencoding containing the tokens ids check if notes is a prettymidi object or not if yes then extract the attributes and put them into a numpy array to round up all the values to the closest int values truncation encodedbatchtokenids for i in rangelennotes encodedbatchtokenids append self encodeplus notesi truncationstrategytruncationstrategy maxlengthmaxlength kwargs tokenids return batchencodingtokenids encodedbatchtokenids def call self notes union np ndarray listprettymidi note listlistprettymidi note padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none padtomultipleof optionalint none returnattentionmask optionalbool none returntensors optionalunionstr tensortype none verbose bool true kwargs batchencoding r this is the call method for pop2pianotokenizer it converts the midi notes to the transformer generated token ids args notes numpy ndarray of shape batchsize maxsequencelength 4 or list of prettymidi note objects this represents the midi notes if notes is a numpy ndarray each sequence must have 4 values they are onset idx offset idx pitch and velocity if notes is a list containing prettymidi note objects each sequence must have 4 attributes they are start end pitch and velocity padding bool str or fileutils paddingstrategy optional defaults to false activates and controls padding accepts the following values true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided maxlength pad to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided false or donotpad default no padding i e can output a batch with sequences of different lengths truncation bool str or tokenizationutilsbase truncationstrategy optional defaults to false activates and controls truncation accepts the following values true or longestfirst truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will truncate token by token removing a token from the longest sequence in the pair if a pair of sequences or a batch of pairs is provided onlyfirst truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the first sequence of a pair if a pair of sequences or a batch of pairs is provided onlysecond truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the second sequence of a pair if a pair of sequences or a batch of pairs is provided false or donottruncate default no truncation i e can output batch with sequence lengths greater than the model maximum admissible input size maxlength int optional controls the maximum length to use by one of the truncationpadding parameters if left unset or set to none this will use the predefined model maximum length if a maximum length is required by one of the truncationpadding parameters if the model has no specific maximum input length like xlnet truncationpadding to a maximum length will be deactivated padtomultipleof int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta returnattentionmask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the returnoutputs attribute what are attention masks glossaryattentionmask returntensors str or fileutils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects verbose bool optional defaults to true whether or not to print more information and warnings returns batchencoding containing the tokenids check if it is batched or not it is batched if its a list containing a list of prettymidi notes where the outer list contains all the batches and the inner list contains all notes for a single batch otherwise if np ndarray is passed it will be considered batched if it has shape of batchsize seqencelength 4 or ndim3 get the truncation and padding strategy if the user has not explicitly mentioned returnattentionmask as false we change it to true since we already have truncated sequnences we are just left to do padding check if they have attentionmasksattentionmask attentionmaskbeatsteps attentionmaskextrapolatedbeatstep or not attentionmaskspresent bool hasattrfeatureextractoroutput attentionmask and hasattrfeatureextractoroutput attentionmaskbeatsteps and hasattrfeatureextractoroutput attentionmaskextrapolatedbeatstep if we are processing batched inputs then we must need attentionmasks if not attentionmaskspresent and featureextractoroutputbeatsteps shape0 1 raise valueerror attentionmask attentionmaskbeatsteps and attentionmaskextrapolatedbeatstep must be present for batched inputs but one of them were not present check for length mismatch between inputsembeds beatsteps and extrapolatedbeatstep if attentionmaskspresent since we know about the number of examples in tokenids from attentionmask if sumfeatureextractoroutputattentionmask 0 0 featureextractoroutputbeatsteps shape0 or featureextractoroutputbeatsteps shape0 featureextractoroutputextrapolatedbeatstep shape0 raise valueerror length mistamtch between tokenids beatsteps and extrapolatedbeatstep found ftokenids length tokenids shape0 beatsteps shape featureextractoroutput beatsteps shape0 fand extrapolatedbeatsteps shape featureextractoroutput extrapolatedbeatstep shape0 if featureextractoroutputattentionmask shape0 tokenids shape0 raise valueerror ffound attentionmask of length featureextractoroutput attentionmask shape0 but tokenids of length tokenids shape0 else if there is no attention mask present then it s surely a single example if featureextractoroutputbeatsteps shape0 1 or featureextractoroutputextrapolatedbeatstep shape0 1 raise valueerror length mistamtch of beatsteps and extrapolatedbeatstep since attentionmask is not present the number of examples must be 1 fbut found beatsteps length featureextractoroutput beatsteps shape0 extrapolatedbeatsteps length featureextractoroutput extrapolatedbeatstep shape0 if attentionmaskspresent check for zerossince tokenids are seperated by zero arrays batchidx np wherefeatureextractoroutputattentionmask 0 00 else batchidx tokenids shape0 noteslist prettymidiobjectslist startidx 0 for index endidx in enumeratebatchidx eachtokensids tokenidsstartidx endidx check where the whole example ended by searching for eostokenid and getting the upper bound eachtokensids eachtokensids np maxnp whereeachtokensids intself eostoken1 1 beatsteps featureextractoroutputbeatstepsindex extrapolatedbeatstep featureextractoroutputextrapolatedbeatstepindex if attention mask is present then mask out real arraytensor if attentionmaskspresent attentionmaskbeatsteps featureextractoroutputattentionmaskbeatstepsindex attentionmaskextrapolatedbeatstep featureextractoroutput attentionmaskextrapolatedbeatstep index beatsteps beatsteps np maxnp whereattentionmaskbeatsteps 10 1 extrapolatedbeatstep extrapolatedbeatstep np maxnp whereattentionmaskextrapolatedbeatstep 10 1 eachtokensids tonumpyeachtokensids beatsteps tonumpybeatsteps extrapolatedbeatstep tonumpyextrapolatedbeatstep prettymidiobject self relativebatchtokensidstomidi tokenseachtokensids beatstepextrapolatedbeatstep barsperbatchself numbars cutofftimeidxself numbars 1 4 for note in prettymidiobject instruments0 notes note start beatsteps0 note end beatsteps0 noteslist appendnote prettymidiobjectslist appendprettymidiobject startidx endidx 1 1 represents the zero array if returnmidi return batchencodingnotes noteslist prettymidiobjects prettymidiobjectslist return batchencodingnotes noteslist coding utf 8 2023 the pop2piano s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for pop2piano offset with onset time shift after previous note_on constructs a pop2piano tokenizer this tokenizer does not require training this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab str path to the vocab file which contains the vocabulary default_velocity int optional defaults to 77 determines the default velocity to be used while creating midi notes num_bars int optional defaults to 2 determines cutoff_time_idx in for each token load the vocab create mappings for encoder returns the vocabulary size of the tokenizer returns the vocabulary of the tokenizer decodes the token ids generated by the transformer into notes args token_id int this denotes the ids generated by the transformers to be converted to midi tokens returns list a list consists of token_type str and value int encodes the midi tokens to transformer generated token ids args token int this denotes the token value token_type str this denotes the type of the token there are four types of midi tokens such as token_time token_velocity token_note and token_special returns int returns the id of the token converts relative tokens to notes which are then used to generate pretty midi object args tokens numpy ndarray tokens to be converted to notes beat_offset_idx int denotes beat offset index for each note in generated midi bars_per_batch int a parameter to control the midi output generation cutoff_time_idx int denotes the cutoff time index for each note in generated midi converts tokens to midi this method calls relative_batch_tokens_ids_to_notes method to convert batch tokens to notes then uses notes_to_midi method to convert them to midi args tokens numpy ndarray denotes tokens which alongside beatstep will be converted to midi beatstep np ndarray we get beatstep from feature extractor which is also used to get midi beat_offset_idx int optional defaults to 0 denotes beat offset index for each note in generated midi bars_per_batch int optional defaults to 2 a parameter to control the midi output generation cutoff_time_idx int optional defaults to 12 denotes the cutoff time index for each note in generated midi taken from the original code please see https github com sweetcocoa pop2piano blob fac11e8dcfc73487513f4588e8d0c22a22f2fdc5 midi_tokenizer py l257 converts relative tokens to notes which will then be used to create pretty midi objects args tokens numpy ndarray relative tokens which will be converted to notes start_idx float a parameter which denotes the starting index cutoff_time_idx float optional a parameter used while converting tokens to notes force offset if no offset for each pitch converts notes to midi args notes numpy ndarray this is used to create pretty midi objects beatstep numpy ndarray this is the extrapolated beatstep that we get from feature extractor offset_sec int optional defaults to 0 0 this represents the offset seconds which is used while creating each pretty midi note saves the tokenizer s vocabulary dictionary to the provided save_directory args save_directory str a path to the directory where to saved it will be created if it doesn t exist filename_prefix optional str optional a prefix to add to the names of the files saved by the tokenizer save the encoder this is the encode_plus method for pop2pianotokenizer it converts the midi notes to the transformer generated token ids it only works on a single batch to process multiple batches please use batch_encode_plus or __call__ method args notes numpy ndarray of shape sequence_length 4 or list of pretty_midi note objects this represents the midi notes if notes is a numpy ndarray each sequence must have 4 values they are onset idx offset idx pitch and velocity if notes is a list containing pretty_midi note objects each sequence must have 4 attributes they are start end pitch and velocity truncation_strategy tokenization_utils_base truncationstrategy optional indicates the truncation strategy that is going to be used during truncation max_length int optional maximum length of the returned list and optionally padding length see above returns batchencoding containing the tokens ids check if notes is a pretty_midi object or not if yes then extract the attributes and put them into a numpy array to round up all the values to the closest int values truncation this is the batch_encode_plus method for pop2pianotokenizer it converts the midi notes to the transformer generated token ids it works on multiple batches by calling encode_plus multiple times in a loop args notes numpy ndarray of shape batch_size sequence_length 4 or list of pretty_midi note objects this represents the midi notes if notes is a numpy ndarray each sequence must have 4 values they are onset idx offset idx pitch and velocity if notes is a list containing pretty_midi note objects each sequence must have 4 attributes they are start end pitch and velocity truncation_strategy tokenization_utils_base truncationstrategy optional indicates the truncation strategy that is going to be used during truncation max_length int optional maximum length of the returned list and optionally padding length see above returns batchencoding containing the tokens ids this is the __call__ method for pop2pianotokenizer it converts the midi notes to the transformer generated token ids args notes numpy ndarray of shape batch_size max_sequence_length 4 or list of pretty_midi note objects this represents the midi notes if notes is a numpy ndarray each sequence must have 4 values they are onset idx offset idx pitch and velocity if notes is a list containing pretty_midi note objects each sequence must have 4 attributes they are start end pitch and velocity padding bool str or file_utils paddingstrategy optional defaults to false activates and controls padding accepts the following values true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths truncation bool str or tokenization_utils_base truncationstrategy optional defaults to false activates and controls truncation accepts the following values true or longest_first truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will truncate token by token removing a token from the longest sequence in the pair if a pair of sequences or a batch of pairs is provided only_first truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the first sequence of a pair if a pair of sequences or a batch of pairs is provided only_second truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the second sequence of a pair if a pair of sequences or a batch of pairs is provided false or do_not_truncate default no truncation i e can output batch with sequence lengths greater than the model maximum admissible input size max_length int optional controls the maximum length to use by one of the truncation padding parameters if left unset or set to none this will use the predefined model maximum length if a maximum length is required by one of the truncation padding parameters if the model has no specific maximum input length like xlnet truncation padding to a maximum length will be deactivated pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the return_outputs attribute what are attention masks glossary attention mask return_tensors str or file_utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects verbose bool optional defaults to true whether or not to print more information and warnings returns batchencoding containing the token_ids check if it is batched or not it is batched if its a list containing a list of pretty_midi notes where the outer list contains all the batches and the inner list contains all notes for a single batch otherwise if np ndarray is passed it will be considered batched if it has shape of batch_size seqence_length 4 or ndim 3 get the truncation and padding strategy if the user has not explicitly mentioned return_attention_mask as false we change it to true since we already have truncated sequnences we are just left to do padding this is the batch_decode method for pop2pianotokenizer it converts the token_ids generated by the transformer to midi_notes and returns them args token_ids union np ndarray torch tensor tf tensor output token_ids of pop2pianoconditionalgeneration model feature_extractor_output batchfeature denotes the output of pop2pianofeatureextractor __call__ it must contain beatstep and extrapolated_beatstep also attention_mask_beatsteps and attention_mask_extrapolated_beatstep should be present if they were returned by the feature extractor return_midi bool optional defaults to true whether to return midi object or not returns if return_midi is true batchencoding containing both notes and pretty_midi pretty_midi prettymidi objects if return_midi is false batchencoding containing notes check if they have attention_masks attention_mask attention_mask_beatsteps attention_mask_extrapolated_beatstep or not if we are processing batched inputs then we must need attention_masks check for length mismatch between inputs_embeds beatsteps and extrapolated_beatstep since we know about the number of examples in token_ids from attention_mask if there is no attention mask present then it s surely a single example check for zeros since token_ids are seperated by zero arrays check where the whole example ended by searching for eos_token_id and getting the upper bound if attention mask is present then mask out real array tensor 1 represents the zero array
import json
import os
from typing import List, Optional, Tuple, Union

import numpy as np

from ...feature_extraction_utils import BatchFeature
from ...tokenization_utils import AddedToken, BatchEncoding, PaddingStrategy, PreTrainedTokenizer, TruncationStrategy
from ...utils import TensorType, is_pretty_midi_available, logging, requires_backends, to_numpy


if is_pretty_midi_available():
    import pretty_midi

logger = logging.get_logger(__name__)


VOCAB_FILES_NAMES = {
    "vocab": "vocab.json",
}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab": {
        "sweetcocoa/pop2piano": "https://huggingface.co/sweetcocoa/pop2piano/blob/main/vocab.json",
    },
}


def token_time_to_note(number, cutoff_time_idx, current_idx):
    current_idx += number
    if cutoff_time_idx is not None:
        current_idx = min(current_idx, cutoff_time_idx)

    return current_idx


def token_note_to_note(number, current_velocity, default_velocity, note_onsets_ready, current_idx, notes):
    if note_onsets_ready[number] is not None:
        
        onset_idx = note_onsets_ready[number]
        if onset_idx < current_idx:
            
            offset_idx = current_idx
            notes.append([onset_idx, offset_idx, number, default_velocity])
            onsets_ready = None if current_velocity == 0 else current_idx
            note_onsets_ready[number] = onsets_ready
    else:
        note_onsets_ready[number] = current_idx
    return notes


class Pop2PianoTokenizer(PreTrainedTokenizer):
    

    model_input_names = ["token_ids", "attention_mask"]
    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP

    def __init__(
        self,
        vocab,
        default_velocity=77,
        num_bars=2,
        unk_token="-1",
        eos_token="1",
        pad_token="0",
        bos_token="2",
        **kwargs,
    ):
        unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token
        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token

        self.default_velocity = default_velocity
        self.num_bars = num_bars

        
        with open(vocab, "rb") as file:
            self.encoder = json.load(file)

        
        self.decoder = {v: k for k, v in self.encoder.items()}

        super().__init__(
            unk_token=unk_token,
            eos_token=eos_token,
            pad_token=pad_token,
            bos_token=bos_token,
            **kwargs,
        )

    @property
    def vocab_size(self):
        
        return len(self.encoder)

    def get_vocab(self):
        
        return dict(self.encoder, **self.added_tokens_encoder)

    def _convert_id_to_token(self, token_id: int) -> list:
        

        token_type_value = self.decoder.get(token_id, f"{self.unk_token}_TOKEN_TIME")
        token_type_value = token_type_value.split("_")
        token_type, value = "_".join(token_type_value[1:]), int(token_type_value[0])

        return [token_type, value]

    def _convert_token_to_id(self, token, token_type="TOKEN_TIME") -> int:
        
        return self.encoder.get(f"{token}_{token_type}", int(self.unk_token))

    def relative_batch_tokens_ids_to_notes(
        self,
        tokens: np.ndarray,
        beat_offset_idx: int,
        bars_per_batch: int,
        cutoff_time_idx: int,
    ):
        

        notes = None

        for index in range(len(tokens)):
            _tokens = tokens[index]
            _start_idx = beat_offset_idx + index * bars_per_batch * 4
            _cutoff_time_idx = cutoff_time_idx + _start_idx
            _notes = self.relative_tokens_ids_to_notes(
                _tokens,
                start_idx=_start_idx,
                cutoff_time_idx=_cutoff_time_idx,
            )

            if len(_notes) == 0:
                pass
            elif notes is None:
                notes = _notes
            else:
                notes = np.concatenate((notes, _notes), axis=0)

        if notes is None:
            return []
        return notes

    def relative_batch_tokens_ids_to_midi(
        self,
        tokens: np.ndarray,
        beatstep: np.ndarray,
        beat_offset_idx: int = 0,
        bars_per_batch: int = 2,
        cutoff_time_idx: int = 12,
    ):
        
        beat_offset_idx = 0 if beat_offset_idx is None else beat_offset_idx
        notes = self.relative_batch_tokens_ids_to_notes(
            tokens=tokens,
            beat_offset_idx=beat_offset_idx,
            bars_per_batch=bars_per_batch,
            cutoff_time_idx=cutoff_time_idx,
        )
        midi = self.notes_to_midi(notes, beatstep, offset_sec=beatstep[beat_offset_idx])
        return midi

    
    
    def relative_tokens_ids_to_notes(self, tokens: np.ndarray, start_idx: float, cutoff_time_idx: float = None):
        
        words = [self._convert_id_to_token(token) for token in tokens]

        current_idx = start_idx
        current_velocity = 0
        note_onsets_ready = [None for i in range(sum([k.endswith("NOTE") for k in self.encoder.keys()]) + 1)]
        notes = []
        for token_type, number in words:
            if token_type == "TOKEN_SPECIAL":
                if number == 1:
                    break
            elif token_type == "TOKEN_TIME":
                current_idx = token_time_to_note(
                    number=number, cutoff_time_idx=cutoff_time_idx, current_idx=current_idx
                )
            elif token_type == "TOKEN_VELOCITY":
                current_velocity = number

            elif token_type == "TOKEN_NOTE":
                notes = token_note_to_note(
                    number=number,
                    current_velocity=current_velocity,
                    default_velocity=self.default_velocity,
                    note_onsets_ready=note_onsets_ready,
                    current_idx=current_idx,
                    notes=notes,
                )
            else:
                raise ValueError("Token type not understood!")

        for pitch, note_onset in enumerate(note_onsets_ready):
            
            if note_onset is not None:
                if cutoff_time_idx is None:
                    cutoff = note_onset + 1
                else:
                    cutoff = max(cutoff_time_idx, note_onset + 1)

                offset_idx = max(current_idx, cutoff)
                notes.append([note_onset, offset_idx, pitch, self.default_velocity])

        if len(notes) == 0:
            return []
        else:
            notes = np.array(notes)
            note_order = notes[:, 0] * 128 + notes[:, 1]
            notes = notes[note_order.argsort()]
            return notes

    def notes_to_midi(self, notes: np.ndarray, beatstep: np.ndarray, offset_sec: int = 0.0):
        

        requires_backends(self, ["pretty_midi"])

        new_pm = pretty_midi.PrettyMIDI(resolution=384, initial_tempo=120.0)
        new_inst = pretty_midi.Instrument(program=0)
        new_notes = []

        for onset_idx, offset_idx, pitch, velocity in notes:
            new_note = pretty_midi.Note(
                velocity=velocity,
                pitch=pitch,
                start=beatstep[onset_idx] - offset_sec,
                end=beatstep[offset_idx] - offset_sec,
            )
            new_notes.append(new_note)
        new_inst.notes = new_notes
        new_pm.instruments.append(new_inst)
        new_pm.remove_invalid_notes()
        return new_pm

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return

        
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab"]
        )
        with open(out_vocab_file, "w") as file:
            file.write(json.dumps(self.encoder))

        return (out_vocab_file,)

    def encode_plus(
        self,
        notes: Union[np.ndarray, List[pretty_midi.Note]],
        truncation_strategy: Optional[TruncationStrategy] = None,
        max_length: Optional[int] = None,
        **kwargs,
    ) -> BatchEncoding:
        r

        requires_backends(self, ["pretty_midi"])

        
        
        if isinstance(notes[0], pretty_midi.Note):
            notes = np.array(
                [[each_note.start, each_note.end, each_note.pitch, each_note.velocity] for each_note in notes]
            ).reshape(-1, 4)

        
        notes = np.round(notes).astype(np.int32)
        max_time_idx = notes[:, :2].max()

        times = [[] for i in range((max_time_idx + 1))]
        for onset, offset, pitch, velocity in notes:
            times[onset].append([pitch, velocity])
            times[offset].append([pitch, 0])

        tokens = []
        current_velocity = 0
        for i, time in enumerate(times):
            if len(time) == 0:
                continue
            tokens.append(self._convert_token_to_id(i, "TOKEN_TIME"))
            for pitch, velocity in time:
                velocity = int(velocity > 0)
                if current_velocity != velocity:
                    current_velocity = velocity
                    tokens.append(self._convert_token_to_id(velocity, "TOKEN_VELOCITY"))
                tokens.append(self._convert_token_to_id(pitch, "TOKEN_NOTE"))

        total_len = len(tokens)

        
        if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and total_len > max_length:
            tokens, _, _ = self.truncate_sequences(
                ids=tokens,
                num_tokens_to_remove=total_len - max_length,
                truncation_strategy=truncation_strategy,
                **kwargs,
            )

        return BatchEncoding({"token_ids": tokens})

    def batch_encode_plus(
        self,
        notes: Union[np.ndarray, List[pretty_midi.Note]],
        truncation_strategy: Optional[TruncationStrategy] = None,
        max_length: Optional[int] = None,
        **kwargs,
    ) -> BatchEncoding:
        r

        encoded_batch_token_ids = []
        for i in range(len(notes)):
            encoded_batch_token_ids.append(
                self.encode_plus(
                    notes[i],
                    truncation_strategy=truncation_strategy,
                    max_length=max_length,
                    **kwargs,
                )["token_ids"]
            )

        return BatchEncoding({"token_ids": encoded_batch_token_ids})

    def __call__(
        self,
        notes: Union[
            np.ndarray,
            List[pretty_midi.Note],
            List[List[pretty_midi.Note]],
        ],
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        r

        
        
        
        
        is_batched = notes.ndim == 3 if isinstance(notes, np.ndarray) else isinstance(notes[0], list)

        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        if is_batched:
            
            return_attention_mask = True if return_attention_mask is None else return_attention_mask
            token_ids = self.batch_encode_plus(
                notes=notes,
                truncation_strategy=truncation_strategy,
                max_length=max_length,
                **kwargs,
            )
        else:
            token_ids = self.encode_plus(
                notes=notes,
                truncation_strategy=truncation_strategy,
                max_length=max_length,
                **kwargs,
            )

        
        token_ids = self.pad(
            token_ids,
            padding=padding_strategy,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            return_tensors=return_tensors,
            verbose=verbose,
        )

        return token_ids

    def batch_decode(
        self,
        token_ids,
        feature_extractor_output: BatchFeature,
        return_midi: bool = True,
    ):
        r

        
        attention_masks_present = bool(
            hasattr(feature_extractor_output, "attention_mask")
            and hasattr(feature_extractor_output, "attention_mask_beatsteps")
            and hasattr(feature_extractor_output, "attention_mask_extrapolated_beatstep")
        )

        
        if not attention_masks_present and feature_extractor_output["beatsteps"].shape[0] > 1:
            raise ValueError(
                "attention_mask, attention_mask_beatsteps and attention_mask_extrapolated_beatstep must be present "
                "for batched inputs! But one of them were not present."
            )

        
        if attention_masks_present:
            
            if (
                sum(feature_extractor_output["attention_mask"][:, 0] == 0)
                != feature_extractor_output["beatsteps"].shape[0]
                or feature_extractor_output["beatsteps"].shape[0]
                != feature_extractor_output["extrapolated_beatstep"].shape[0]
            ):
                raise ValueError(
                    "Length mistamtch between token_ids, beatsteps and extrapolated_beatstep! Found "
                    f"token_ids length - {token_ids.shape[0]}, beatsteps shape - {feature_extractor_output['beatsteps'].shape[0]} "
                    f"and extrapolated_beatsteps shape - {feature_extractor_output['extrapolated_beatstep'].shape[0]}"
                )
            if feature_extractor_output["attention_mask"].shape[0] != token_ids.shape[0]:
                raise ValueError(
                    f"Found attention_mask of length - {feature_extractor_output['attention_mask'].shape[0]} but token_ids of length - {token_ids.shape[0]}"
                )
        else:
            
            if (
                feature_extractor_output["beatsteps"].shape[0] != 1
                or feature_extractor_output["extrapolated_beatstep"].shape[0] != 1
            ):
                raise ValueError(
                    "Length mistamtch of beatsteps and extrapolated_beatstep! Since attention_mask is not present the number of examples must be 1, "
                    f"But found beatsteps length - {feature_extractor_output['beatsteps'].shape[0]}, extrapolated_beatsteps length - {feature_extractor_output['extrapolated_beatstep'].shape[0]}."
                )

        if attention_masks_present:
            
            batch_idx = np.where(feature_extractor_output["attention_mask"][:, 0] == 0)[0]
        else:
            batch_idx = [token_ids.shape[0]]

        notes_list = []
        pretty_midi_objects_list = []
        start_idx = 0
        for index, end_idx in enumerate(batch_idx):
            each_tokens_ids = token_ids[start_idx:end_idx]
            
            each_tokens_ids = each_tokens_ids[:, : np.max(np.where(each_tokens_ids == int(self.eos_token))[1]) + 1]
            beatsteps = feature_extractor_output["beatsteps"][index]
            extrapolated_beatstep = feature_extractor_output["extrapolated_beatstep"][index]

            
            if attention_masks_present:
                attention_mask_beatsteps = feature_extractor_output["attention_mask_beatsteps"][index]
                attention_mask_extrapolated_beatstep = feature_extractor_output[
                    "attention_mask_extrapolated_beatstep"
                ][index]
                beatsteps = beatsteps[: np.max(np.where(attention_mask_beatsteps == 1)[0]) + 1]
                extrapolated_beatstep = extrapolated_beatstep[
                    : np.max(np.where(attention_mask_extrapolated_beatstep == 1)[0]) + 1
                ]

            each_tokens_ids = to_numpy(each_tokens_ids)
            beatsteps = to_numpy(beatsteps)
            extrapolated_beatstep = to_numpy(extrapolated_beatstep)

            pretty_midi_object = self.relative_batch_tokens_ids_to_midi(
                tokens=each_tokens_ids,
                beatstep=extrapolated_beatstep,
                bars_per_batch=self.num_bars,
                cutoff_time_idx=(self.num_bars + 1) * 4,
            )

            for note in pretty_midi_object.instruments[0].notes:
                note.start += beatsteps[0]
                note.end += beatsteps[0]
                notes_list.append(note)

            pretty_midi_objects_list.append(pretty_midi_object)
            start_idx += end_idx + 1  

        if return_midi:
            return BatchEncoding({"notes": notes_list, "pretty_midi_objects": pretty_midi_objects_list})

        return BatchEncoding({"notes": notes_list})