codingutf8 2023 the facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for speecht5 import os from shutil import copyfile from typing import any dict list optional tuple import sentencepiece as spm from tokenizationutils import pretrainedtokenizer from utils import logging from numbernormalizer import englishnumbernormalizer logger logging getloggername vocabfilesnames vocabfile spmchar model pretrainedvocabfilesmap vocabfile microsoftspeecht5asr https huggingface comicrosoftspeecht5asrresolvemainspmchar model microsoftspeecht5tts https huggingface comicrosoftspeecht5ttsresolvemainspmchar model microsoftspeecht5vc https huggingface comicrosoftspeecht5vcresolvemainspmchar model pretrainedpositionalembeddingssizes microsoftspeecht5asr 1024 microsoftspeecht5tts 1024 microsoftspeecht5vc 1024 class speecht5tokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask def init self vocabfile bostokens eostokens unktokenunk padtokenpad normalizefalse spmodelkwargs optionaldictstr any none kwargs none self spmodelkwargs if spmodelkwargs is none else spmodelkwargs self vocabfile vocabfile self normalize normalize self normalizer none self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadvocabfile super init bostokenbostoken eostokeneostoken unktokenunktoken padtokenpadtoken normalizenormalize spmodelkwargsself spmodelkwargs kwargs def preparefortokenizationself text issplitintowordsfalse kwargs normalize kwargs popnormalize self normalize if issplitintowords text text if normalize text self normalizertext return text kwargs property def vocabsizeself return self spmodel getpiecesize property def normalizerself if self normalizer is none self normalizer englishnumbernormalizer return self normalizer normalizer setter def normalizerself value self normalizer value def getvocabself vocab self convertidstotokensi i for i in rangeself vocabsize vocab updateself addedtokensencoder return vocab def getstateself state self dict copy statespmodel none return state def setstateself d self dict d for backward compatibility if not hasattrself spmodelkwargs self spmodelkwargs self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadself vocabfile def tokenizeself text str liststr converts a token str in an id using the vocab return self spmodel piecetoidtoken def convertidtotokenself index converts a sequence of tokens string in a single string currentsubtokens outstring for token in tokens make sure that special tokens are not decoded using sentencepiece model if token in self allspecialtokens outstring self spmodel decodecurrentsubtokens token currentsubtokens else currentsubtokens appendtoken outstring self spmodel decodecurrentsubtokens return outstring strip def buildinputswithspecialtokensself tokenids0 tokenids1none listint we don t expect to process pairs but leave the pair logic for api consistency coding utf 8 2023 the facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for speecht5 construct a speecht5 tokenizer based on sentencepiece https github com google sentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer bos_token str optional defaults to s the begin of sequence token eos_token str optional defaults to s the end of sequence token unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths normalize bool optional defaults to false whether to convert numeric quantities in the text to their spelt out english counterparts sp_model_kwargs dict optional will be passed to the sentencepieceprocessor __init__ method the python wrapper for sentencepiece https github com google sentencepiece tree master python can be used among other things to set enable_sampling enable subword regularization nbest_size sampling parameters for unigram invalid for bpe dropout nbest_size 0 1 no sampling is performed nbest_size 1 samples from the nbest_size results nbest_size 0 assuming that nbest_size is infinite and samples from the all hypothesis lattice using forward filtering and backward sampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpe dropout attributes sp_model sentencepieceprocessor the sentencepiece processor that is used for every conversion string tokens and ids for backward compatibility take as input a string and return a list of strings tokens for words sub words converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string make sure that special tokens are not decoded using sentencepiece model build model inputs from a sequence by appending eos_token_id we don t expect to process pairs but leave the pair logic for api consistency
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from .number_normalizer import EnglishNumberNormalizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "spm_char.model"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "microsoft/speecht5_asr": "https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model",
        "microsoft/speecht5_tts": "https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model",
        "microsoft/speecht5_vc": "https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model",
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "microsoft/speecht5_asr": 1024,
    "microsoft/speecht5_tts": 1024,
    "microsoft/speecht5_vc": 1024,
}


class SpeechT5Tokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        bos_token="<s>",
        eos_token="</s>",
        unk_token="<unk>",
        pad_token="<pad>",
        normalize=False,
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ) -> None:
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
        self.vocab_file = vocab_file
        self.normalize = normalize
        self._normalizer = None

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            pad_token=pad_token,
            normalize=normalize,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
        normalize = kwargs.pop("normalize", self.normalize)
        if is_split_into_words:
            text = " " + text
        if normalize:
            text = self.normalizer(text)
        return (text, kwargs)

    @property
    def vocab_size(self):
        return self.sp_model.get_piece_size()

    @property
    def normalizer(self):
        if self._normalizer is None:
            self._normalizer = EnglishNumberNormalizer()
        return self._normalizer

    @normalizer.setter
    def normalizer(self, value):
        self._normalizer = value

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    def _tokenize(self, text: str) -> List[str]:
        
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        
        token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        
        current_sub_tokens = []
        out_string = ""
        for token in tokens:
            
            if token in self.all_special_tokens:
                out_string += self.sp_model.decode(current_sub_tokens) + token
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string.strip()

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        
        if token_ids_1 is None:
            return token_ids_0 + [self.eos_token_id]
        
        return token_ids_0 + token_ids_1 + [self.eos_token_id]

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        suffix_ones = [1]
        if token_ids_1 is None:
            return ([0] * len(token_ids_0)) + suffix_ones
        return ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)