codingutf8 2023 the kakao enterprise s the mmstts s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for vits import json import os import re from typing import any dict list optional tuple union from tokenizationutils import pretrainedtokenizer from utils import isphonemizeravailable logging if isphonemizeravailable import phonemizer logger logging getloggername vocabfilesnames vocabfile vocab json pretrainedvocabfilesmap vocabfile facebookmmsttseng https huggingface cofacebookmmsttsengresolvemainvocab json pretrainedpositionalembeddingssizes this model does not have a maximum input length facebookmmsttseng 4096 def hasnonromancharactersinputstring find any character outside the ascii range nonromanpattern re compilerx00x7f search the input string for nonroman characters match nonromanpattern searchinputstring hasnonroman match is not none return hasnonroman class vitstokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask def init self vocabfile padtokenpad unktokenunk languagenone addblanktrue normalizetrue phonemizetrue isuromanfalse kwargs none with openvocabfile encodingutf8 as vocabhandle self encoder json loadvocabhandle self decoder v k for k v in self encoder items self language language self addblank addblank self normalize normalize self phonemize phonemize self isuroman isuroman super init padtokenpadtoken unktokenunktoken languagelanguage addblankaddblank normalizenormalize phonemizephonemize isuromanisuroman kwargs property def vocabsizeself return lenself encoder def getvocabself vocab self convertidstotokensi i for i in rangeself vocabsize vocab updateself addedtokensencoder return vocab def normalizetextself inputstring special treatment of characters in certain languages if self language ron text text replace return text def preparefortokenization self text str issplitintowords bool false normalize optionalbool none kwargs tuplestr dictstr any normalize normalize if normalize is not none else self normalize if normalize normalise for casing text self normalizetexttext filteredtext self preprocesschartext if hasnonromancharactersfilteredtext and self isuroman logger warning text to the tokenizer contains nonroman characters ensure the uroman romanizer is applied to the text prior to passing it to the tokenizer see https github comisinlpuroman for details if self phonemize if not isphonemizeravailable raise importerrorplease install the phonemizer python package to use this tokenizer filteredtext phonemizer phonemize filteredtext languageenus backendespeak striptrue preservepunctuationtrue withstresstrue filteredtext re subrs filteredtext elif normalize strip any chars outside of the vocab punctuation filteredtext joinlistfilterlambda char char in self encoder filteredtext strip return filteredtext kwargs def tokenizeself text str liststr converts a token str in an id using the vocab return self encoder gettoken self encoder getself unktoken def convertidtotokenself index coding utf 8 2023 the kakao enterprise s the mms tts s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for vits this model does not have a maximum input length find any character outside the ascii range search the input string for non roman characters construct a vits tokenizer also supports mms tts this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file language str optional language identifier add_blank bool optional defaults to true whether to insert token id 0 in between the other tokens normalize bool optional defaults to true whether to normalize the input text by removing all casing and punctuation phonemize bool optional defaults to true whether to convert the input text into phonemes is_uroman bool optional defaults to false whether the uroman romanizer needs to be applied to the input text prior to tokenizing lowercase the input string respecting any special token ids that may be part or entirely upper cased special treatment of characters in certain languages performs any necessary transformations before tokenization this method should pop the arguments from kwargs and return the remaining kwargs as well we test the kwargs at the end of the encoding process to be sure all the arguments have been used args text str the text to prepare is_split_into_words bool optional defaults to false whether or not the input is already pre tokenized e g split into words if set to true the tokenizer assumes the input is already split into words for instance by splitting it on whitespace which it will tokenize normalize bool optional defaults to none whether or not to apply punctuation and casing normalization to the text inputs typically vits is trained on lower cased and un punctuated text hence normalization is used to ensure that the input text consists only of lower case characters kwargs dict str any optional keyword arguments to use for the tokenization returns tuple str dict str any the prepared text and the unused kwargs normalise for casing strip any chars outside of the vocab punctuation tokenize a string by inserting the pad token at the boundary between adjacent characters converts a token str in an id using the vocab converts an index integer in a token str using the vocab
import json
import os
import re
from typing import Any, Dict, List, Optional, Tuple, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_phonemizer_available, logging


if is_phonemizer_available():
    import phonemizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "facebook/mms-tts-eng": "https://huggingface.co/facebook/mms-tts-eng/resolve/main/vocab.json",
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    
    "facebook/mms-tts-eng": 4096,
}


def has_non_roman_characters(input_string):
    
    non_roman_pattern = re.compile(r"[^\x00-\x7F]")

    
    match = non_roman_pattern.search(input_string)
    has_non_roman = match is not None
    return has_non_roman


class VitsTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        pad_token="<pad>",
        unk_token="<unk>",
        language=None,
        add_blank=True,
        normalize=True,
        phonemize=True,
        is_uroman=False,
        **kwargs,
    ) -> None:
        with open(vocab_file, encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)

        self.decoder = {v: k for k, v in self.encoder.items()}
        self.language = language
        self.add_blank = add_blank
        self.normalize = normalize
        self.phonemize = phonemize

        self.is_uroman = is_uroman

        super().__init__(
            pad_token=pad_token,
            unk_token=unk_token,
            language=language,
            add_blank=add_blank,
            normalize=normalize,
            phonemize=phonemize,
            is_uroman=is_uroman,
            **kwargs,
        )

    @property
    def vocab_size(self):
        return len(self.encoder)

    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def normalize_text(self, input_string):
        
        all_vocabulary = list(self.encoder.keys()) + list(self.added_tokens_encoder.keys())
        filtered_text = ""

        i = 0
        while i < len(input_string):
            found_match = False
            for word in all_vocabulary:
                if input_string[i : i + len(word)] == word:
                    filtered_text += word
                    i += len(word)
                    found_match = True
                    break

            if not found_match:
                filtered_text += input_string[i].lower()
                i += 1

        return filtered_text

    def _preprocess_char(self, text):
        
        if self.language == "ron":
            text = text.replace("ț", "ţ")
        return text

    def prepare_for_tokenization(
        self, text: str, is_split_into_words: bool = False, normalize: Optional[bool] = None, **kwargs
    ) -> Tuple[str, Dict[str, Any]]:
        
        normalize = normalize if normalize is not None else self.normalize

        if normalize:
            
            text = self.normalize_text(text)

        filtered_text = self._preprocess_char(text)

        if has_non_roman_characters(filtered_text) and self.is_uroman:
            logger.warning(
                "Text to the tokenizer contains non-Roman characters. Ensure the `uroman` Romanizer is "
                "applied to the text prior to passing it to the tokenizer. See "
                "`https://github.com/isi-nlp/uroman` for details."
            )

        if self.phonemize:
            if not is_phonemizer_available():
                raise ImportError("Please install the `phonemizer` Python package to use this tokenizer.")

            filtered_text = phonemizer.phonemize(
                filtered_text,
                language="en-us",
                backend="espeak",
                strip=True,
                preserve_punctuation=True,
                with_stress=True,
            )
            filtered_text = re.sub(r"\s+", " ", filtered_text)
        elif normalize:
            
            filtered_text = "".join(list(filter(lambda char: char in self.encoder, filtered_text))).strip()

        return filtered_text, kwargs

    def _tokenize(self, text: str) -> List[str]:
        
        tokens = list(text)

        if self.add_blank:
            interspersed = [self._convert_id_to_token(0)] * (len(tokens) * 2 + 1)
            interspersed[1::2] = tokens
            tokens = interspersed

        return tokens

    def convert_tokens_to_string(self, tokens: List[str]) -> str:
        if self.add_blank and len(tokens) > 1:
            tokens = tokens[1::2]
        return "".join(tokens)

    def _convert_token_to_id(self, token):
        
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        
        return self.decoder.get(index)

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Union[Tuple[str], None]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return

        vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        with open(vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        return (vocab_file,)