codingutf8 2020 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model pegasus import os from shutil import copyfile from typing import list optional tuple from tokenizationutilsfast import pretrainedtokenizerfast from utils import issentencepieceavailable logging if issentencepieceavailable from tokenizationpegasus import pegasustokenizer else pegasustokenizer none logger logging getloggername spieceunderline vocabfilesnames vocabfile spiece model tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile googlepegasusxsum https huggingface cogooglepegasusxsumresolvemainspiece model tokenizerfile googlepegasusxsum https huggingface cogooglepegasusxsumresolvemaintokenizer json pretrainedpositionalembeddingssizes googlepegasusxsum 512 class pegasustokenizerfastpretrainedtokenizerfast r construct a fast pegasus tokenizer backed by huggingface s tokenizers library based on unigramhttps huggingface codocstokenizerspythonlatestcomponents html highlightunigrammodels this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocabfile str sentencepiecehttps github comgooglesentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer padtoken str optional defaults to pad the token used for padding for example when batching sequences of different lengths eostoken str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the septoken tip unktoken str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead masktoken str optional defaults to mask2 the token used for masking single token values this is the token used when training this model with masked language modeling mlm this is the token that the pegasus encoder will try to predict during pretraining it corresponds to mask2 in pegasus pretraining with extracted gapsentences for abstractive summarizationhttps arxiv orgpdf1912 08777 pdf masktokensent str optional defaults to mask1 the token used for masking whole target sentences this is the token used when training this model with gap sentences generation gsg this is the sentence that the pegasus decoder will try to predict during pretraining it corresponds to mask1 in pegasus pretraining with extracted gapsentences for abstractive summarizationhttps arxiv orgpdf1912 08777 pdf additionalspecialtokens liststr optional additional special tokens used by the tokenizer if no additionalspecialtokens are provided mask2 and unk2 unk102 are used as additional special tokens corresponding to the original pegasus tokenizerhttps github comgoogleresearchpegasusblob939830367bcf411193d2b5eca2f2f90f3f9260capegasusopspretrainparsingops ccl66 that uses the tokens 2 104 only for pretraining fill additional tokens with unktoken102 in case not all additional tokens are already taken pegasus was design to support changing the index of the first tokens if one of the paddingeosunkmask token is different from default we must rebuild the vocab get list where entries are 1 if a token is eos or pad else 0 if alreadyhasspecialtokens return self specialtokenmasktokenids0 elif tokenids1 is none return self specialtokenmasktokenids0 1 else return self specialtokenmasktokenids0 tokenids1 1 def buildinputswithspecialtokensself tokenids0 tokenids1none listint if tokenids1 is none return tokenids0 self eostokenid we don t expect to process pairs but leave the pair logic for api consistency return tokenids0 tokenids1 self eostokenid def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr if not self cansaveslowtokenizer raise valueerror your fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile copyfileself vocabfile outvocabfile return outvocabfile coding utf 8 2020 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization class for model pegasus construct a fast pegasus tokenizer backed by huggingface s tokenizers library based on unigram https huggingface co docs tokenizers python latest components html highlight unigram models this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str sentencepiece https github com google sentencepiece file generally has a spm extension that contains the vocabulary necessary to instantiate a tokenizer pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead mask_token str optional defaults to mask_2 the token used for masking single token values this is the token used when training this model with masked language modeling mlm this is the token that the pegasus encoder will try to predict during pretraining it corresponds to mask2 in pegasus pre training with extracted gap sentences for abstractive summarization https arxiv org pdf 1912 08777 pdf mask_token_sent str optional defaults to mask_1 the token used for masking whole target sentences this is the token used when training this model with gap sentences generation gsg this is the sentence that the pegasus decoder will try to predict during pretraining it corresponds to mask1 in pegasus pre training with extracted gap sentences for abstractive summarization https arxiv org pdf 1912 08777 pdf additional_special_tokens list str optional additional special tokens used by the tokenizer if no additional_special_tokens are provided mask_2 and unk_2 unk_102 are used as additional special tokens corresponding to the original pegasus tokenizer https github com google research pegasus blob 939830367bcf411193d2b5eca2f2f90f3f9260ca pegasus ops pretrain_parsing_ops cc l66 that uses the tokens 2 104 only for pretraining entries 2 104 are only used for pretraining fill additional tokens with unk_token_102 in case not all additional tokens are already taken pegasus was design to support changing the index of the first tokens if one of the padding eos unk mask token is different from default we must rebuild the vocab call it once instead of inside list comp unk is only sometimes special get list where entries are 1 if a token is eos or pad else 0 build model inputs from a sequence by adding eos to the end no bos token is added to the front single sequence x s pair of sequences a b s not intended use args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens we don t expect to process pairs but leave the pair logic for api consistency
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_pegasus import PegasusTokenizer
else:
    PegasusTokenizer = None


logger = logging.get_logger(__name__)


SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"},
    "tokenizer_file": {
        "google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "google/pegasus-xsum": 512,
}


class PegasusTokenizerFast(PreTrainedTokenizerFast):
    r

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = PegasusTokenizer
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        pad_token="<pad>",
        eos_token="</s>",
        unk_token="<unk>",
        mask_token="<mask_2>",
        mask_token_sent="<mask_1>",
        additional_special_tokens=None,
        offset=103,  
        **kwargs,
    ):
        self.offset = offset

        if additional_special_tokens is not None:
            if not isinstance(additional_special_tokens, list):
                raise TypeError(
                    f"additional_special_tokens should be of type {type(list)}, but is"
                    f" {type(additional_special_tokens)}"
                )

            additional_special_tokens_extended = (
                ([mask_token_sent] + additional_special_tokens)
                if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                else additional_special_tokens
            )
            
            additional_special_tokens_extended += [
                f"<unk_{i}>" for i in range(len(additional_special_tokens_extended), self.offset - 1)
            ]

            if len(set(additional_special_tokens_extended)) != len(additional_special_tokens_extended):
                raise ValueError(
                    "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                    f" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."
                )
            additional_special_tokens = additional_special_tokens_extended
        else:
            additional_special_tokens = [mask_token_sent] if mask_token_sent is not None else []
            additional_special_tokens += [f"<unk_{i}>" for i in range(2, self.offset)]

        
        
        from_slow = kwargs.pop("from_slow", None)
        from_slow = from_slow or str(pad_token) != "<pad>" or str(eos_token) != "</s>" or str(unk_token) != "<unk>"

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            pad_token=pad_token,
            eos_token=eos_token,
            unk_token=unk_token,
            mask_token=mask_token,
            mask_token_sent=mask_token_sent,
            offset=offset,
            additional_special_tokens=additional_special_tokens,
            from_slow=from_slow,
            **kwargs,
        )
        self.vocab_file = vocab_file

    @property
    def can_save_slow_tokenizer(self) -> bool:
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    def _special_token_mask(self, seq):
        all_special_ids = set(self.all_special_ids)  
        all_special_ids.remove(self.unk_token_id)  

        if all_special_ids != set(range(len(self.additional_special_tokens) + 3)):
            raise ValueError(
                "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
                f" {len(self.additional_special_tokens)} additional_special_tokens, but got {all_special_ids}"
            )

        return [1 if x in all_special_ids else 0 for x in seq]

    def get_special_tokens_mask(
        self, token_ids_0: List, token_ids_1: Optional[List] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return self._special_token_mask(token_ids_0)
        elif token_ids_1 is None:
            return self._special_token_mask(token_ids_0) + [1]
        else:
            return self._special_token_mask(token_ids_0 + token_ids_1) + [1]

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
        
        if token_ids_1 is None:
            return token_ids_0 + [self.eos_token_id]
        
        return token_ids_0 + token_ids_1 + [self.eos_token_id]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)