codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for roformer import json from typing import list optional tuple from tokenizers import normalizers from tokenizers pretokenizers import bertpretokenizer pretokenizer from tokenizationutilsfast import pretrainedtokenizerfast from utils import logging from tokenizationroformer import roformertokenizer from tokenizationutils import jiebapretokenizer logger logging getloggername vocabfilesnames vocabfile vocab txt tokenizerfile tokenizer json pretrainedvocabfilesmap vocabfile junnyuroformerchinesesmall https huggingface cojunnyuroformerchinesesmallresolvemainvocab txt junnyuroformerchinesebase https huggingface cojunnyuroformerchinesebaseresolvemainvocab txt junnyuroformerchinesecharsmall https huggingface cojunnyuroformerchinesecharsmallresolvemainvocab txt junnyuroformerchinesecharbase https huggingface cojunnyuroformerchinesecharbaseresolvemainvocab txt junnyuroformersmalldiscriminator https huggingface cojunnyuroformersmalldiscriminatorresolvemainvocab txt junnyuroformersmallgenerator https huggingface cojunnyuroformersmallgeneratorresolvemainvocab txt pretrainedpositionalembeddingssizes junnyuroformerchinesesmall 1536 junnyuroformerchinesebase 1536 junnyuroformerchinesecharsmall 512 junnyuroformerchinesecharbase 512 junnyuroformersmalldiscriminator 128 junnyuroformersmallgenerator 128 pretrainedinitconfiguration junnyuroformerchinesesmall dolowercase true junnyuroformerchinesebase dolowercase true junnyuroformerchinesecharsmall dolowercase true junnyuroformerchinesecharbase dolowercase true junnyuroformersmalldiscriminator dolowercase true junnyuroformersmallgenerator dolowercase true class roformertokenizerfastpretrainedtokenizerfast r construct a fast roformer tokenizer backed by huggingface s tokenizers library roformertokenizerfast is almost identical to berttokenizerfast and runs endtoend tokenization punctuation splitting and wordpiece there are some difference between them when tokenizing chinese this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods example python from transformers import roformertokenizerfast tokenizer roformertokenizerfast frompretrainedjunnyuroformerchinesebase tokenizer tokenize vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes pretrainedinitconfiguration pretrainedinitconfiguration slowtokenizerclass roformertokenizer def init self vocabfilenone tokenizerfilenone dolowercasetrue unktokenunk septokensep padtokenpad clstokencls masktokenmask tokenizechinesecharstrue stripaccentsnone kwargs super init vocabfile tokenizerfiletokenizerfile dolowercasedolowercase unktokenunktoken septokenseptoken padtokenpadtoken clstokenclstoken masktokenmasktoken tokenizechinesecharstokenizechinesechars stripaccentsstripaccents kwargs pretokstate json loadsself backendtokenizer normalizer getstate if pretokstate getlowercase dolowercase dolowercase or pretokstate getstripaccents stripaccents stripaccents pretokclass getattrnormalizers pretokstate poptype pretokstatelowercase dolowercase pretokstatestripaccents stripaccents self backendtokenizer normalizer pretokclasspretokstate self dolowercase dolowercase def getstateself state self dict copy statetokenizer pretokenizer bertpretokenizer return state def setstateself d self dict d vocab self dicttokenizer getvocab self dicttokenizer pretokenizer pretokenizer customjiebapretokenizervocab def buildinputswithspecialtokensself tokenids0 tokenids1none output self clstokenid tokenids0 self septokenid if tokenids1 is not none output tokenids1 self septokenid return output def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep 0 lentokenids1 sep 1 def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles def savepretrained self savedirectory legacyformatnone filenameprefixnone pushtohubfalse kwargs self backendtokenizer pretokenizer bertpretokenizer return super savepretrainedsavedirectory legacyformat filenameprefix pushtohub kwargs coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for roformer construct a fast roformer tokenizer backed by huggingface s tokenizers library roformertokenizerfast is almost identical to berttokenizerfast and runs end to end tokenization punctuation splitting and wordpiece there are some difference between them when tokenizing chinese this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods example python from transformers import roformertokenizerfast tokenizer roformertokenizerfast from_pretrained junnyu roformer_chinese_base tokenizer tokenize 今天天气非常好 今 天 天 气 非常 好 build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a roformer sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequence pair classification task a roformer sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "junnyu/roformer_chinese_small": 1536,
    "junnyu/roformer_chinese_base": 1536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


PRETRAINED_INIT_CONFIGURATION = {
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}


class RoFormerTokenizerFast(PreTrainedTokenizerFast):
    r

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    slow_tokenizer_class = RoFormerTokenizer

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        do_lower_case=True,
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        tokenize_chinese_chars=True,
        strip_accents=None,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            do_lower_case=do_lower_case,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            tokenize_chinese_chars=tokenize_chinese_chars,
            strip_accents=strip_accents,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            pre_tok_state.get("lowercase", do_lower_case) != do_lower_case
            or pre_tok_state.get("strip_accents", strip_accents) != strip_accents
        ):
            pre_tok_class = getattr(normalizers, pre_tok_state.pop("type"))
            pre_tok_state["lowercase"] = do_lower_case
            pre_tok_state["strip_accents"] = strip_accents
            self.backend_tokenizer.normalizer = pre_tok_class(**pre_tok_state)

        self.do_lower_case = do_lower_case

    def __getstate__(self):
        state = self.__dict__.copy()
        state["_tokenizer"].pre_tokenizer = BertPreTokenizer()
        return state

    def __setstate__(self, d):
        self.__dict__ = d
        vocab = self.__dict__["_tokenizer"].get_vocab()
        self.__dict__["_tokenizer"].pre_tokenizer = PreTokenizer.custom(JiebaPreTokenizer(vocab))

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        
        output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]

        if token_ids_1 is not None:
            output += token_ids_1 + [self.sep_token_id]

        return output

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    def save_pretrained(
        self,
        save_directory,
        legacy_format=None,
        filename_prefix=None,
        push_to_hub=False,
        **kwargs,
    ):
        self.backend_tokenizer.pre_tokenizer = BertPreTokenizer()
        return super().save_pretrained(save_directory, legacy_format, filename_prefix, push_to_hub, **kwargs)