codingutf8 2020 the trax s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model reformer import os from shutil import copyfile from typing import any dict list optional tuple import sentencepiece as spm from tokenizationutils import pretrainedtokenizer from utils import logging logger logging getloggername spieceunderline vocabfilesnames vocabfile spiece model pretrainedvocabfilesmap vocabfile googlereformercrimeandpunishment https huggingface cogooglereformercrimeandpunishmentresolvemainspiece model pretrainedpositionalembeddingssizes googlereformercrimeandpunishment 524288 class reformertokenizerpretrainedtokenizer vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask def init self vocabfile eostokens unktokenunk additionalspecialtokens spmodelkwargs optionaldictstr any none kwargs none self spmodelkwargs if spmodelkwargs is none else spmodelkwargs self vocabfile vocabfile self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadvocabfile super init eostokeneostoken unktokenunktoken additionalspecialtokensadditionalspecialtokens spmodelkwargsself spmodelkwargs kwargs property def vocabsizeself return self spmodel getpiecesize def getvocabself dictstr int vocab self convertidstotokensi i for i in rangeself vocabsize vocab updateself addedtokensencoder return vocab def getstateself state self dict copy statespmodel none return state def setstateself d self dict d for backward compatibility if not hasattrself spmodelkwargs self spmodelkwargs self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadself vocabfile def tokenizeself text str liststr converts a token str in an id using the vocab return self spmodel piecetoidtoken def convertidtotokenself index converts a sequence of tokens string in a single string currentsubtokens outstring for token in tokens make sure that special tokens are not decoded using sentencepiece model if token in self allspecialtokens outstring self spmodel decodecurrentsubtokens token currentsubtokens else currentsubtokens appendtoken outstring self spmodel decodecurrentsubtokens return outstring strip def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile and os path isfileself vocabfile copyfileself vocabfile outvocabfile elif not os path isfileself vocabfile with openoutvocabfile wb as fi contentspiecemodel self spmodel serializedmodelproto fi writecontentspiecemodel return outvocabfile coding utf 8 2020 the trax s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model reformer construct a reformer tokenizer based on sentencepiece https github com google sentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead additional_special_tokens list str optional defaults to additional special tokens used by the tokenizer sp_model_kwargs dict optional will be passed to the sentencepieceprocessor __init__ method the python wrapper for sentencepiece https github com google sentencepiece tree master python can be used among other things to set enable_sampling enable subword regularization nbest_size sampling parameters for unigram invalid for bpe dropout nbest_size 0 1 no sampling is performed nbest_size 1 samples from the nbest_size results nbest_size 0 assuming that nbest_size is infinite and samples from the all hypothesis lattice using forward filtering and backward sampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpe dropout for backward compatibility take as input a string and return a list of strings tokens for words sub words converts a token str in an id using the vocab converts an index integer in a token str using the vocab converts a sequence of tokens string in a single string make sure that special tokens are not decoded using sentencepiece model
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)


SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/reformer-crime-and-punishment": 524288,
}


class ReformerTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        eos_token="</s>",
        unk_token="<unk>",
        additional_special_tokens=[],
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ) -> None:
        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.vocab_file = vocab_file
        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(vocab_file)

        super().__init__(
            eos_token=eos_token,
            unk_token=unk_token,
            additional_special_tokens=additional_special_tokens,
            sp_model_kwargs=self.sp_model_kwargs,
            **kwargs,
        )

    @property
    def vocab_size(self):
        return self.sp_model.get_piece_size()

    def get_vocab(self) -> Dict[str, int]:
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(self.vocab_file)

    def _tokenize(self, text: str) -> List[str]:
        
        return self.sp_model.encode(text, out_type=str)

    def _convert_token_to_id(self, token):
        
        return self.sp_model.piece_to_id(token)

    def _convert_id_to_token(self, index):
        
        if index < self.sp_model.get_piece_size():
            token = self.sp_model.IdToPiece(index)
        return token

    def convert_tokens_to_string(self, tokens):
        
        current_sub_tokens = []
        out_string = ""
        for token in tokens:
            
            if token in self.all_special_tokens:
                out_string += self.sp_model.decode(current_sub_tokens) + token
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
        out_string += self.sp_model.decode(current_sub_tokens)
        return out_string.strip()

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        return (out_vocab_file,)