codingutf8 2020 the trax s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model reformer import os from shutil import copyfile from typing import optional tuple from tokenizationutilsfast import pretrainedtokenizerfast from utils import issentencepieceavailable logging if issentencepieceavailable from tokenizationreformer import reformertokenizer else reformertokenizer none logger logging getloggername spieceunderline vocabfilesnames vocabfile spiece model tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile googlereformercrimeandpunishment https huggingface cogooglereformercrimeandpunishmentresolvemainspiece model tokenizerfile googlereformercrimeandpunishment https huggingface cogooglereformercrimeandpunishmentresolvemaintokenizer json pretrainedpositionalembeddingssizes googlereformercrimeandpunishment 524288 class reformertokenizerfastpretrainedtokenizerfast vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass reformertokenizer def init self vocabfilenone tokenizerfilenone eostokens unktokenunk additionalspecialtokens kwargs super init vocabfile tokenizerfiletokenizerfile eostokeneostoken unktokenunktoken additionalspecialtokensadditionalspecialtokens kwargs self vocabfile vocabfile property def cansaveslowtokenizerself bool return os path isfileself vocabfile if self vocabfile else false def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not self cansaveslowtokenizer raise valueerror your fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile copyfileself vocabfile outvocabfile return outvocabfile coding utf 8 2020 the trax s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model reformer construct a fast reformer tokenizer backed by huggingface s tokenizers library based on unigram https huggingface co docs tokenizers python latest components html highlight unigram models this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths additional_special_tokens list str optional additional special tokens used by the tokenizer
import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_reformer import ReformerTokenizer
else:
    ReformerTokenizer = None


logger = logging.get_logger(__name__)


SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    },
    "tokenizer_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json"
        )
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/reformer-crime-and-punishment": 524288,
}


class ReformerTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = ReformerTokenizer

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        eos_token="</s>",
        unk_token="<unk>",
        additional_special_tokens=[],
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            eos_token=eos_token,
            unk_token=unk_token,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

        self.vocab_file = vocab_file

    @property
    def can_save_slow_tokenizer(self) -> bool:
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)