codingutf8 google ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for canine from typing import dict list optional from tokenizationutils import addedtoken pretrainedtokenizer from utils import logging logger logging getloggername pretrainedpositionalembeddingssizes nielsrcanines 2048 unicode defines 1 114 112 total codepoints unicodevocabsize 1114112 below constants defining canonical codepoints for special pseudocharacters copied from https github comgoogleresearchlanguageblobmasterlanguagecaninespecialcodepoints py pad 0 cls 0xe000 sep 0xe001 bos 0xe002 mask 0xe003 reserved 0xe004 maps special codepoints to humanreadable names specialcodepoints dictint str special symbols are represented using codepoints values that are valid but designated as private use meaning that they will never be assigned characters by the unicode consortium and are thus safe for use here note do not add any sort of unkchar here they are explicitly excluded and should fail with a hard error cls cls sep sep bos bos mask mask pad pad reserved reserved maps special codepoint humanreadable names to their codepoint values specialcodepointsbyname dictstr int name codepoint for codepoint name in specialcodepoints items class caninetokenizerpretrainedtokenizer r construct a canine tokenizer i e a character splitter it turns text into a sequence of characters and then converts each character into its unicode code point caninetokenizer inherits from pretrainedtokenizer refer to superclass pretrainedtokenizer for usage examples and documentation concerning parameters args modelmaxlength int optional defaults to 2048 the maximum sentence length the model accepts mask token behave like a normal word i e include the space before it creates a mapping for looking up the ids of special symbols creates a mapping for looking up the string forms of special symbol ids tokenize a string i e perform character splitting return listtext def converttokentoidself token str int converts a unicode code point integer in a token str in case it s a special code point convert to humanreadable format build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a canine sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args tokenids0 listint list of ids to which the special tokens will be added tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of input ids glossaryinputids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepareformodel method args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs alreadyhasspecialtokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns listint a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequencepair classification task a canine sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if tokenids1 is none this method only returns the first portion of the mask 0s args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of token type ids glossarytokentypeids according to the given sequences caninetokenizer has no vocab file coding utf 8 google ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for canine unicode defines 1 114 112 total codepoints below constants defining canonical codepoints for special pseudo characters copied from https github com google research language blob master language canine special_codepoints py maps special codepoints to human readable names special symbols are represented using codepoints values that are valid but designated as private use meaning that they will never be assigned characters by the unicode consortium and are thus safe for use here note do not add any sort of unk_char here they are explicitly excluded and should fail with a hard error maps special codepoint human readable names to their codepoint values construct a canine tokenizer i e a character splitter it turns text into a sequence of characters and then converts each character into its unicode code point caninetokenizer inherits from pretrainedtokenizer refer to superclass pretrainedtokenizer for usage examples and documentation concerning parameters args model_max_length int optional defaults to 2048 the maximum sentence length the model accepts mask token behave like a normal word i e include the space before it creates a mapping for looking up the ids of special symbols creates a mapping for looking up the string forms of special symbol ids tokenize a string i e perform character splitting converts a token i e a unicode character in an id i e its integer unicode code point value converts a unicode code point integer in a token str in case it s a special code point convert to human readable format build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a canine sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token create a mask from the two sequences passed to be used in a sequence pair classification task a canine sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s caninetokenizer has no vocab file
from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


logger = logging.get_logger(__name__)


PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "nielsr/canine-s": 2048,
}


UNICODE_VOCAB_SIZE = 1114112



PAD = 0
CLS = 0xE000
SEP = 0xE001
BOS = 0xE002
MASK = 0xE003
RESERVED = 0xE004


SPECIAL_CODEPOINTS: Dict[int, str] = {
    
    
    
    
    
    
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}


SPECIAL_CODEPOINTS_BY_NAME: Dict[str, int] = {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}


class CanineTokenizer(PreTrainedTokenizer):
    r

    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

    def __init__(
        self,
        bos_token=chr(CLS),
        eos_token=chr(SEP),
        sep_token=chr(SEP),
        cls_token=chr(CLS),
        pad_token=chr(PAD),
        mask_token=chr(MASK),
        add_prefix_space=False,
        model_max_length=2048,
        **kwargs,
    ):
        bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token
        eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token
        sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token
        cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token
        pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token

        
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        
        self._special_codepoints: Dict[str, int] = {}
        for codepoint, name in SPECIAL_CODEPOINTS.items():
            self._special_codepoints[name] = codepoint

        
        self._special_codepoint_strings: Dict[int, str] = {
            codepoint: name for name, codepoint in self._special_codepoints.items()
        }

        self._unicode_vocab_size = UNICODE_VOCAB_SIZE
        self._num_special_tokens = len(self._special_codepoints)

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            pad_token=pad_token,
            mask_token=mask_token,
            add_prefix_space=add_prefix_space,
            model_max_length=model_max_length,
            **kwargs,
        )

    @property
    def vocab_size(self) -> int:
        return self._unicode_vocab_size

    def get_vocab(self):
        vocab = {chr(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    def _tokenize(self, text: str) -> List[str]:
        
        return list(text)

    def _convert_token_to_id(self, token: str) -> int:
        
        try:
            return ord(token)
        except TypeError:
            raise ValueError(f"invalid token: '{token}'")

    def _convert_id_to_token(self, index: int) -> str:
        
        try:
            if index in SPECIAL_CODEPOINTS:
                return SPECIAL_CODEPOINTS[index]
            return chr(index)
        except TypeError:
            raise ValueError(f"invalid id: {index}")

    def convert_tokens_to_string(self, tokens):
        return "".join(tokens)

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        result = cls + token_ids_0 + sep
        if token_ids_1 is not None:
            result += token_ids_1 + sep
        return result

    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        
        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        result = [1] + ([0] * len(token_ids_0)) + [1]
        if token_ids_1 is not None:
            result += ([0] * len(token_ids_1)) + [1]
        return result

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        result = len(cls + token_ids_0 + sep) * [0]
        if token_ids_1 is not None:
            result += len(token_ids_1 + sep) * [1]
        return result

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
        return ()