


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__					:     Any       =     logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__					:     Tuple       =     '▁'

SCREAMING_SNAKE_CASE__					:     Optional[int]       =     {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE__					:     Dict       =     {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

SCREAMING_SNAKE_CASE__					:     Dict       =     {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

SCREAMING_SNAKE_CASE__					:     Dict       =     {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

SCREAMING_SNAKE_CASE__					:     Any       =     {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}


class 		_SCREAMING_SNAKE_CASE     (						A     ):
     __SCREAMING_SNAKE_CASE     					=    ["input_ids"]

     __SCREAMING_SNAKE_CASE     					=    VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_INIT_CONFIGURATION
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE     					=    RESOURCE_FILES_NAMES


     def __init__(  self							,					A_							,					A_=None							,					A_=False							,					A_="utf8"							,					A_="[UNK]"							,					A_="[SEP]"							,					A_="[PAD]"							,					A_="[CLS]"							,					A_="[MASK]"							,					A_ = None							,					**A_							,					):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.

       _UpperCAmelCase		:							int							=     {} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=A_							,					unk_token=A_							,					sep_token=A_							,					pad_token=A_							,					cls_token=A_							,					mask_token=A_							,					vocab_file=A_							,					encoding=A_							,					sp_model_kwargs=self.sp_model_kwargs							,					**A_							,					)
       _UpperCAmelCase		:							List[str]							=     do_lower_case
       _UpperCAmelCase		:							Optional[int]							=     sentencepiece_model_ckpt
       _UpperCAmelCase		:							Union[str, Any]							=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(A_    )

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
         _UpperCAmelCase		:							Tuple							=     self.load_vocab(filepath=A_    )
       else:
         _UpperCAmelCase		:							Optional[int]							=     {self.sp_model.id_to_piece(A_    ): id for id in range(self.sp_model.get_piece_size()    )}
       _UpperCAmelCase		:							Optional[int]							=     {v: k for k, v in self.vocab.items()}


     def        __snake_case(  self							,					A_    ):
       if text is None:
         return None

       _UpperCAmelCase		:							Tuple							=     self.tokenize(A_    )
       _UpperCAmelCase,_UpperCAmelCase		:							Optional[int]							=     """""", []

       for i, ch in enumerate(A_    ):
         if ch in self.SP_CHAR_MAPPING:
           _UpperCAmelCase		:							int							=     self.SP_CHAR_MAPPING.get(A_    )
         else:
           _UpperCAmelCase		:							Any							=     unicodedata.normalize("""NFKC"""							,					A_    )
         if self.is_whitespace(A_    ):
           continue
         normalized_text += ch
         char_mapping.extend([i] * len(A_    )    )

       _UpperCAmelCase,_UpperCAmelCase,_UpperCAmelCase		:							Any							=     normalized_text, [], 0

       if self.do_lower_case:
         _UpperCAmelCase		:							Optional[Any]							=     text.lower()

       for token in split_tokens:
         if token[:1] == "▁":
           _UpperCAmelCase		:							str							=     token[1:]
         _UpperCAmelCase		:							Optional[int]							=     text[offset:].index(A_    ) + offset
         _UpperCAmelCase		:							Any							=     start + len(A_    )

         token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
         _UpperCAmelCase		:							Dict							=     end
       return token_mapping


     @property
     def        __snake_case(  self    ):
       return len(self.vocab    )


     def        __snake_case(  self    ):
       return dict(self.vocab							,					**self.added_tokens_encoder    )


     def __getstate__(  self    ):
       _UpperCAmelCase		:							Any							=     self.__dict__.copy()
       _UpperCAmelCase		:							Dict							=     None
       return state


     def __setstate__(  self							,					A_    ):
       _UpperCAmelCase		:							Union[str, Any]							=     d

       # for backward compatibility
       if not hasattr(self							,					"""sp_model_kwargs"""    ):
         _UpperCAmelCase		:							Dict							=     {}

       _UpperCAmelCase		:							Tuple							=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(self.sentencepiece_model_ckpt    )


     def        __snake_case(  self							,					A_    ):
       return "".join((self.SP_CHAR_MAPPING.get(A_							,					A_    ) for c in text)    )


     def        __snake_case(  self							,					A_							,					A_=False							,					A_=64							,					A_=0.1    ):

       if self.sp_model_kwargs.get("""enable_sampling"""    ) is True:
         _UpperCAmelCase		:							Optional[Any]							=     True
       if self.sp_model_kwargs.get("""alpha"""    ) is not None:
         _UpperCAmelCase		:							Dict							=     self.sp_model_kwargs.get("""alpha"""    )
       if self.sp_model_kwargs.get("""nbest_size"""    ) is not None:
         _UpperCAmelCase		:							List[Any]							=     self.sp_model_kwargs.get("""nbest_size"""    )

       if not enable_sampling:
         _UpperCAmelCase		:							str							=     self.sp_model.EncodeAsPieces(A_    )
       else:
         _UpperCAmelCase		:							Any							=     self.sp_model.SampleEncodeAsPieces(A_							,					A_							,					A_    )
       _UpperCAmelCase		:							Optional[int]							=     []
       for pi, piece in enumerate(A_    ):
         if piece == SPIECE_UNDERLINE:
           if not pieces[pi + 1].startswith(A_    ) and pi != 0:
             new_pieces.append(A_    )
             continue
           else:
             continue
         _UpperCAmelCase		:							Optional[Any]							=     0
         for i, chunk in enumerate(A_    ):
           if chunk == SPIECE_UNDERLINE:
             continue
           if self.is_ch_char(A_    ) or self.is_punct(A_    ):
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
               new_pieces.append(piece[lst_i:i]    )
             new_pieces.append(A_    )
             _UpperCAmelCase		:							str							=     i + 1
           elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
               new_pieces.append(piece[lst_i:i]    )
             _UpperCAmelCase		:							List[Any]							=     i
           elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
               new_pieces.append(piece[lst_i:i]    )
             _UpperCAmelCase		:							Tuple							=     i
         if len(A_    ) > lst_i:
           new_pieces.append(piece[lst_i:]    )
       return new_pieces


     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							Tuple							=     """""".join(A_    ).replace(A_							,					""" """    ).strip()
       return out_string


     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							Optional[int]							=     self.convert_ids_to_tokens(A_    )
       _UpperCAmelCase		:							Dict							=     """""".join(A_    ).replace(A_							,					""" """    ).strip()
       return out_string


     def        __snake_case(  self							,					A_    ):
       return self.vocab.get(A_							,					self.vocab.get(self.unk_token    )    )


     def        __snake_case(  self							,					A_    ):
       return self.reverse_vocab.get(A_							,					self.unk_token    )


     def        __snake_case(  self							,					A_							,					A_=None    ):
       if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       _UpperCAmelCase		:							List[str]							=     [self.cls_token_id]
       _UpperCAmelCase		:							Dict							=     [self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


     def        __snake_case(  self							,					A_							,					A_=None    ):
       if offset_mapping_a is None:
         return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


     def        __snake_case(  self							,					A_							,					A_=None							,					A_=False    ):

       if already_has_special_tokens:
         if token_ids_a is not None:
           raise ValueError(
               """You should not supply a second sequence if the provided sequence of """
               """ids is already formatted with special tokens for the model."""    )
         return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
         return [1] + ([0] * len(A_    )) + [1, 1] + ([0] * len(A_    )) + [1]
       return [1] + ([0] * len(A_    )) + [1]


     def        __snake_case(  self							,					A_							,					A_ = None    ):
       # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
       if token_ids_a is None:
         # [CLS] X [SEP]
         return (len(A_    ) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(A_    ) + 1) + [1] * (len(A_    ) + 3)


     def        __snake_case(  self							,					A_    ):
       if "\u4e00" <= char <= "\u9fff":
         return True
       return False


     def        __snake_case(  self							,					A_    ):
       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
         return True
       return False


     def        __snake_case(  self							,					A_    ):
       if char in ",;:.?!~，；：。？！《》【】":
         return True
       return False


     def        __snake_case(  self							,					A_    ):
       if char == " " or char == "\t" or char == "\n" or char == "\r":
         return True
       if len(A_    ) == 1:
         _UpperCAmelCase		:							int							=     unicodedata.category(A_    )
         if cat == "Zs":
           return True
       return False


     def        __snake_case(  self							,					A_    ):
       _UpperCAmelCase		:							Dict							=     {}
       with io.open(A_							,					"""r"""							,					encoding="""utf-8"""    ) as f:
         for index, line in enumerate(A_    ):
           _UpperCAmelCase		:							Dict							=     line.rstrip("""\n"""    )
           _UpperCAmelCase		:							str							=     int(A_    )

       return token_to_idx




     def        __snake_case(  self							,					A_							,					A_ = None    ):
       _UpperCAmelCase		:							Optional[int]							=     0
       if os.path.isdir(A_    ):
         _UpperCAmelCase		:							Optional[Any]							=     os.path.join(
             A_							,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
       else:
         _UpperCAmelCase		:							Union[str, Any]							=     (filename_prefix + """-""" if filename_prefix else """""") + save_directory
       with open(A_							,					"""w"""							,					encoding="""utf-8"""    ) as writer:
         for token, token_index in sorted(self.vocab.items()							,					key=lambda A_    : kv[1]    ):
           if index != token_index:
             logger.warning(
                 F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                 """ Please check that the vocabulary is not corrupted!"""    )
             _UpperCAmelCase		:							Any							=     token_index
           writer.write(token + """\n"""    )
           index += 1

       _UpperCAmelCase		:							Tuple							=     os.path.join(A_							,					"""sentencepiece.bpe.model"""    )
       with open(A_							,					"""wb"""    ) as fi:
         _UpperCAmelCase		:							Tuple							=     self.sp_model.serialized_model_proto()
         fi.write(A_    )

       return (vocab_file,)









import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


SCREAMING_SNAKE_CASE__					:     List[str]       =     logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__					:     Optional[int]       =     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE__					:     List[Any]       =     {
    'vocab_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'yjernite/retribert-base-uncased': (
            'https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

SCREAMING_SNAKE_CASE__					:     Tuple       =     {
    'yjernite/retribert-base-uncased': 512,
}


SCREAMING_SNAKE_CASE__					:     Optional[Any]       =     {
    'yjernite/retribert-base-uncased': {'do_lower_case': True},
}


class 		_SCREAMING_SNAKE_CASE     (						A     ):
     __SCREAMING_SNAKE_CASE     					=    VOCAB_FILES_NAMES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_VOCAB_FILES_MAP
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __SCREAMING_SNAKE_CASE     					=    PRETRAINED_INIT_CONFIGURATION
     __SCREAMING_SNAKE_CASE     					=    RetriBertTokenizer
     __SCREAMING_SNAKE_CASE     					=    ['''input_ids''', '''attention_mask''']


     def __init__(  self							,					A_=None							,					A_=None							,					A_=True							,					A_="[UNK]"							,					A_="[SEP]"							,					A_="[PAD]"							,					A_="[CLS]"							,					A_="[MASK]"							,					A_=True							,					A_=None							,					**A_							,					):
       super().__init__(
           A_							,					tokenizer_file=A_							,					do_lower_case=A_							,					unk_token=A_							,					sep_token=A_							,					pad_token=A_							,					cls_token=A_							,					mask_token=A_							,					tokenize_chinese_chars=A_							,					strip_accents=A_							,					**A_							,					)

       _UpperCAmelCase		:							List[str]							=     json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
       if (
           normalizer_state.get("""lowercase"""							,					A_    ) != do_lower_case
           or normalizer_state.get("""strip_accents"""							,					A_    ) != strip_accents
           or normalizer_state.get("""handle_chinese_chars"""							,					A_    ) != tokenize_chinese_chars
       ):
         _UpperCAmelCase		:							str							=     getattr(A_							,					normalizer_state.pop("""type"""    )    )
         _UpperCAmelCase		:							Optional[int]							=     do_lower_case
         _UpperCAmelCase		:							List[Any]							=     strip_accents
         _UpperCAmelCase		:							List[str]							=     tokenize_chinese_chars
         _UpperCAmelCase		:							Dict							=     normalizer_class(**A_    )

       _UpperCAmelCase		:							Dict							=     do_lower_case


     def        __snake_case(  self							,					A_							,					A_=None    ):
       _UpperCAmelCase		:							Tuple							=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output


     def        __snake_case(  self							,					A_							,					A_ = None    ):
       _UpperCAmelCase		:							List[Any]							=     [self.sep_token_id]
       _UpperCAmelCase		:							Tuple							=     [self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




     def        __snake_case(  self							,					A_							,					A_ = None    ):
       _UpperCAmelCase		:							Dict							=     self._tokenizer.model.save(A_							,					name=A_    )
       return tuple(A_    )





