codingutf8 2020 the google ai language team s allegro pl facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license construct a fast bpe tokenizer for herbert backed by huggingface s tokenizers library peculiarities uses bert s pretokenizer bertpretokenizer splits tokens on spaces and also on punctuation each occurrence of a punctuation character will be treated separately this tokenizer inherits from pretrainedtokenizer which contains most of the methods users should refer to the superclass for more information regarding methods args vocabfile str path to the vocabulary file mergesfile str path to the merges file build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an herbert like bert sequence has the following format single sequence s x s pair of sequences s a s b s args tokenids0 listint list of ids to which the special tokens will be added tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of input ids glossaryinputids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepareformodel method args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs alreadyhasspecialtokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns listint a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequencepair classification task herbert like bert sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of token type ids glossarytokentypeids according to the given sequences coding utf 8 2020 the google ai language team s allegro pl facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license construct a fast bpe tokenizer for herbert backed by huggingface s tokenizers library peculiarities uses bert s pre tokenizer bertpretokenizer splits tokens on spaces and also on punctuation each occurrence of a punctuation character will be treated separately this tokenizer inherits from pretrainedtokenizer which contains most of the methods users should refer to the superclass for more information regarding methods args vocab_file str path to the vocabulary file merges_file str path to the merges file build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an herbert like bert sequence has the following format single sequence s x s pair of sequences s a s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequence pair classification task herbert like bert sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {"allegro/herbert-base-cased": 514}
PRETRAINED_INIT_CONFIGURATION = {}


class HerbertTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = HerbertTokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        sep_token="</s>",
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            sep_token=sep_token,
            **kwargs,
        )

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        if token_ids_1 is None:
            return cls + token_ids_0 + sep

        return cls + token_ids_0 + sep + token_ids_1 + sep

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is None:
            return [1] + ([0] * len(token_ids_0)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)