codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization utils for roformer from typing import list from tokenizers import normalizedstring pretokenizedstring normalizers class jiebapretokenizer def initself vocab none self vocab vocab self normalizers normalizers bertnormalizer cleantextfalse handlechinesecharstrue stripaccentsfalse lowercasefalse try import rjieba except importerror raise importerror you need to install rjieba to use roformertokenizer see https pypi orgprojectrjieba for installation self jieba rjieba def jiebasplitself i int normalizedstring normalizedstring listnormalizedstring splits this code slice normalizedstring is too slow 6s but testalignementmethods can pass for token start end in self jieba tokenizestrnormalizedstring hmmfalse if token in self vocab splits appendnormalizedstringstart end else tokenlist self normalizers normalizestrtoken split for token in tokenlist if token end start lentoken splits appendnormalizedstringstart end start end this code testalignementmethods can t pass but fast 300ms for token in self jieba cutstrnormalizedstring false if token in self vocab splits appendnormalizedstringtoken else tokenlist self normalizers normalizestrtoken split for token in tokenlist if token splits appendnormalizedstringtoken return splits def pretokenizeself pretok pretokenizedstring pretok splitself jiebasplit coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization utils for roformer this code slice normalized_string is too slow 6s but test_alignement_methods can pass this code test_alignement_methods can t pass but fast 300ms for token in self jieba cut str normalized_string false if token in self vocab splits append normalizedstring token else token_list self normalizers normalize_str token split for token in token_list if token splits append normalizedstring token
from typing import List

from tokenizers import NormalizedString, PreTokenizedString, normalizers


class JiebaPreTokenizer:
    def __init__(self, vocab) -> None:
        self.vocab = vocab
        self.normalizers = normalizers.BertNormalizer(
            clean_text=False,
            handle_chinese_chars=True,
            strip_accents=False,
            lowercase=False,
        )
        try:
            import rjieba
        except ImportError:
            raise ImportError(
                "You need to install rjieba to use RoFormerTokenizer. "
                "See https://pypi.org/project/rjieba/ for installation."
            )
        self.jieba = rjieba

    def jieba_split(self, i: int, normalized_string: NormalizedString) -> List[NormalizedString]:
        splits = []

        
        for token, start, end in self.jieba.tokenize(str(normalized_string), hmm=False):
            if token in self.vocab:
                splits.append(normalized_string[start:end])
            else:
                token_list = self.normalizers.normalize_str(token).split()
                for token in token_list:
                    if token:
                        end = start + len(token)
                        splits.append(normalized_string[start:end])
                        start = end

        
        
        
        
        
        
        
        
        

        return splits

    def pre_tokenize(self, pretok: PreTokenizedString):
        pretok.split(self.jieba_split)