

"""simple docstring"""


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase													=logging.get_logger(__name__)

UpperCAmelCase													="▁"

UpperCAmelCase													={"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

UpperCAmelCase													={
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

UpperCAmelCase													={
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

UpperCAmelCase													={
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

UpperCAmelCase													={
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   ["input_ids"]

    _lowerCamelCase					=   VOCAB_FILES_NAMES
    _lowerCamelCase					=   PRETRAINED_INIT_CONFIGURATION
    _lowerCamelCase					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCamelCase					=   PRETRAINED_VOCAB_FILES_MAP
    _lowerCamelCase					=   RESOURCE_FILES_NAMES




    def __init__( self ,lowerCamelCase_ ,lowerCamelCase_=None ,lowerCamelCase_=False ,lowerCamelCase_="utf8" ,lowerCamelCase_="[UNK]" ,lowerCamelCase_="[SEP]" ,lowerCamelCase_="[PAD]" ,lowerCamelCase_="[CLS]" ,lowerCamelCase_="[MASK]" ,lowerCamelCase_ = None ,**lowerCamelCase_ ,)    ->      None:
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.

          A				  =		{} if sp_model_kwargs is None else sp_model_kwargs
          super().__init__(
              do_lower_case=lowerCamelCase_ ,unk_token=lowerCamelCase_ ,sep_token=lowerCamelCase_ ,pad_token=lowerCamelCase_ ,cls_token=lowerCamelCase_ ,mask_token=lowerCamelCase_ ,vocab_file=lowerCamelCase_ ,encoding=lowerCamelCase_ ,sp_model_kwargs=self.sp_model_kwargs ,**lowerCamelCase_ ,)
          A				  =		do_lower_case
          A				  =		sentencepiece_model_ckpt
          A				  =		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
          self.sp_model.Load(lowerCamelCase_    )

          # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
          if vocab_file is not None:
                A				  =		self.load_vocab(filepath=lowerCamelCase_    )
          else:
                A				  =		{self.sp_model.id_to_piece(lowerCamelCase_    ): id for id in range(self.sp_model.get_piece_size()    )}
          A				  =		{v: k for k, v in self.vocab.items()}




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      str:
          if text is None:
                return None

          A				  =		self.tokenize(lowerCamelCase_    )
          A  ,			A				  =		"""""", []

          for i, ch in enumerate(lowerCamelCase_    ):
                if ch in self.SP_CHAR_MAPPING:
                      A				  =		self.SP_CHAR_MAPPING.get(lowerCamelCase_    )
                else:
                      A				  =		unicodedata.normalize("""NFKC""" ,lowerCamelCase_    )
                if self.is_whitespace(lowerCamelCase_    ):
                      continue
                normalized_text += ch
                char_mapping.extend([i] * len(lowerCamelCase_    )    )

          A  ,			A  ,			A				  =		normalized_text, [], 0

          if self.do_lower_case:
                A				  =		text.lower()

          for token in split_tokens:
                if token[:1] == "▁":
                      A				  =		token[1:]
                A				  =		text[offset:].index(lowerCamelCase_    ) + offset
                A				  =		start + len(lowerCamelCase_    )

                token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
                A				  =		end
          return token_mapping




    @property
    def UpperCamelCase__		( self    )    ->      Dict:
          return len(self.vocab    )




    def UpperCamelCase__		( self    )    ->      Any:
          return dict(self.vocab ,**self.added_tokens_encoder    )




    def __getstate__( self    )    ->      Union[str, Any]:
          A				  =		self.__dict__.copy()
          A				  =		None
          return state




    def __setstate__( self ,lowerCamelCase_    )    ->      Optional[Any]:
          A				  =		d

          # for backward compatibility
          if not hasattr(self ,"""sp_model_kwargs"""    ):
                A				  =		{}

          A				  =		spm.SentencePieceProcessor(**self.sp_model_kwargs    )
          self.sp_model.Load(self.sentencepiece_model_ckpt    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[Any]:
          return "".join((self.SP_CHAR_MAPPING.get(lowerCamelCase_ ,lowerCamelCase_    ) for c in text)    )




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=False ,lowerCamelCase_=6_4 ,lowerCamelCase_=0.1    )    ->      int:

          if self.sp_model_kwargs.get("""enable_sampling"""    ) is True:
                A				  =		True
          if self.sp_model_kwargs.get("""alpha"""    ) is not None:
                A				  =		self.sp_model_kwargs.get("""alpha"""    )
          if self.sp_model_kwargs.get("""nbest_size"""    ) is not None:
                A				  =		self.sp_model_kwargs.get("""nbest_size"""    )

          if not enable_sampling:
                A				  =		self.sp_model.EncodeAsPieces(lowerCamelCase_    )
          else:
                A				  =		self.sp_model.SampleEncodeAsPieces(lowerCamelCase_ ,lowerCamelCase_ ,lowerCamelCase_    )
          A				  =		[]
          for pi, piece in enumerate(lowerCamelCase_    ):
                if piece == SPIECE_UNDERLINE:
                      if not pieces[pi + 1].startswith(lowerCamelCase_    ) and pi != 0:
                            new_pieces.append(lowerCamelCase_    )
                            continue
                      else:
                            continue
                A				  =		0
                for i, chunk in enumerate(lowerCamelCase_    ):
                      if chunk == SPIECE_UNDERLINE:
                            continue
                      if self.is_ch_char(lowerCamelCase_    ) or self.is_punct(lowerCamelCase_    ):
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]    )
                            new_pieces.append(lowerCamelCase_    )
                            A				  =		i + 1
                      elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]    )
                            A				  =		i
                      elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]    )
                            A				  =		i
                if len(lowerCamelCase_    ) > lst_i:
                      new_pieces.append(piece[lst_i:]    )
          return new_pieces




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Tuple:
          A				  =		"""""".join(lowerCamelCase_    ).replace(lowerCamelCase_ ,""" """    ).strip()
          return out_string




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Tuple:
          A				  =		self.convert_ids_to_tokens(lowerCamelCase_    )
          A				  =		"""""".join(lowerCamelCase_    ).replace(lowerCamelCase_ ,""" """    ).strip()
          return out_string




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Dict:
          return self.vocab.get(lowerCamelCase_ ,self.vocab.get(self.unk_token    )    )




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      Optional[Any]:
          return self.reverse_vocab.get(lowerCamelCase_ ,self.unk_token    )




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None    )    ->      List[str]:
          if token_ids_a is None:
                return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          A				  =		[self.cls_token_id]
          A				  =		[self.sep_token_id]
          return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None    )    ->      Any:
          if offset_mapping_a is None:
                return [(0, 0)] + offset_mapping_a + [(0, 0)]

          return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None ,lowerCamelCase_=False    )    ->      Optional[Any]:

          if already_has_special_tokens:
                if token_ids_a is not None:
                      raise ValueError(
                          """You should not supply a second sequence if the provided sequence of """
                          """ids is already formatted with special tokens for the model."""    )
                return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

          if token_ids_a is not None:
                return [1] + ([0] * len(lowerCamelCase_    )) + [1, 1] + ([0] * len(lowerCamelCase_    )) + [1]
          return [1] + ([0] * len(lowerCamelCase_    )) + [1]




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      List[int]:
          # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
          if token_ids_a is None:
                # [CLS] X [SEP]
                return (len(lowerCamelCase_    ) + 2) * [0]

          # [CLS] A [SEP] [SEP] B [SEP]
          return [0] * (len(lowerCamelCase_    ) + 1) + [1] * (len(lowerCamelCase_    ) + 3)




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[Any]:
          if "\u4e00" <= char <= "\u9fff":
                return True
          return False




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[str]:
          if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                return True
          return False




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[str]:
          if char in ",;:.?!~，；：。？！《》【】":
                return True
          return False




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      List[str]:
          if char == " " or char == "\t" or char == "\n" or char == "\r":
                return True
          if len(lowerCamelCase_    ) == 1:
                A				  =		unicodedata.category(lowerCamelCase_    )
                if cat == "Zs":
                      return True
          return False




    def UpperCamelCase__		( self ,lowerCamelCase_    )    ->      str:
          A				  =		{}
          with io.open(lowerCamelCase_ ,"""r""" ,encoding="""utf-8"""    ) as f:
                for index, line in enumerate(lowerCamelCase_    ):
                      A				  =		line.rstrip("""\n"""    )
                      A				  =		int(lowerCamelCase_    )

          return token_to_idx




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      Tuple[str]:
          A				  =		0
          if os.path.isdir(lowerCamelCase_    ):
                A				  =		os.path.join(
                    lowerCamelCase_ ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
          else:
                A				  =		(filename_prefix + """-""" if filename_prefix else """""") + save_directory
          with open(lowerCamelCase_ ,"""w""" ,encoding="""utf-8"""    ) as writer:
                for token, token_index in sorted(self.vocab.items() ,key=lambda lowerCamelCase_    : kv[1]    ):
                      if index != token_index:
                            logger.warning(
                                f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                """ Please check that the vocabulary is not corrupted!"""    )
                            A				  =		token_index
                      writer.write(token + """\n"""    )
                      index += 1

          A				  =		os.path.join(lowerCamelCase_ ,"""sentencepiece.bpe.model"""    )
          with open(lowerCamelCase_ ,"""wb"""    ) as fi:
                A				  =		self.sp_model.serialized_model_proto()
                fi.write(lowerCamelCase_    )

          return (vocab_file,)







"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase													=logging.get_logger(__name__)

UpperCAmelCase													={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase													=[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCAmelCase													={
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCAmelCase													={f"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase													={f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   VOCAB_FILES_NAMES
    _lowerCamelCase					=   PRETRAINED_VOCAB_FILES_MAP
    _lowerCamelCase					=   PRETRAINED_INIT_CONFIGURATION
    _lowerCamelCase					=   FunnelTokenizer
    _lowerCamelCase					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCamelCase					=   2




    def __init__( self ,lowerCamelCase_=None ,lowerCamelCase_=None ,lowerCamelCase_=True ,lowerCamelCase_="<unk>" ,lowerCamelCase_="<sep>" ,lowerCamelCase_="<pad>" ,lowerCamelCase_="<cls>" ,lowerCamelCase_="<mask>" ,lowerCamelCase_="<s>" ,lowerCamelCase_="</s>" ,lowerCamelCase_=True ,lowerCamelCase_=True ,lowerCamelCase_=None ,lowerCamelCase_="##" ,**lowerCamelCase_ ,)    ->      List[Any]:
          super().__init__(
              lowerCamelCase_ ,tokenizer_file=lowerCamelCase_ ,do_lower_case=lowerCamelCase_ ,unk_token=lowerCamelCase_ ,sep_token=lowerCamelCase_ ,pad_token=lowerCamelCase_ ,cls_token=lowerCamelCase_ ,mask_token=lowerCamelCase_ ,bos_token=lowerCamelCase_ ,eos_token=lowerCamelCase_ ,clean_text=lowerCamelCase_ ,tokenize_chinese_chars=lowerCamelCase_ ,strip_accents=lowerCamelCase_ ,wordpieces_prefix=lowerCamelCase_ ,**lowerCamelCase_ ,)

          A				  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
          if (
              normalizer_state.get("""lowercase""" ,lowerCamelCase_    ) != do_lower_case
              or normalizer_state.get("""strip_accents""" ,lowerCamelCase_    ) != strip_accents
              or normalizer_state.get("""handle_chinese_chars""" ,lowerCamelCase_    ) != tokenize_chinese_chars
          ):
                A				  =		getattr(lowerCamelCase_ ,normalizer_state.pop("""type"""    )    )
                A				  =		do_lower_case
                A				  =		strip_accents
                A				  =		tokenize_chinese_chars
                A				  =		normalizer_class(**lowerCamelCase_    )

          A				  =		do_lower_case




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None    )    ->      List[Any]:
          A				  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      List[int]:
          A				  =		[self.sep_token_id]
          A				  =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0]
          return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      Tuple[str]:
          A				  =		self._tokenizer.model.save(lowerCamelCase_ ,name=lowerCamelCase_    )
          return tuple(lowerCamelCase_    )




