





"""simple docstring"""




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_     :						Union[str, Any]						=					logging.get_logger(__name__)

lowerCamelCase_     :						List[str]						=					'▁'

lowerCamelCase_     :						Any						=					{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

lowerCamelCase_     :						List[Any]						=					{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

lowerCamelCase_     :						Union[str, Any]						=					{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

lowerCamelCase_     :						Optional[int]						=					{
    'ernie-m-base': 5_14,
    'ernie-m-large': 5_14,
}

lowerCamelCase_     :						Dict						=					{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}

class  _UpperCAmelCase  (   UpperCAmelCase__		):



      '''simple docstring'''
      lowercase_      : List[str]	    =   ["input_ids"]

      lowercase_      : Optional[Any]	    =   VOCAB_FILES_NAMES
      lowercase_      : Dict	    =   PRETRAINED_INIT_CONFIGURATION
      lowercase_      : List[Any]	    =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowercase_      : Union[str, Any]	    =   PRETRAINED_VOCAB_FILES_MAP
      lowercase_      : Optional[Any]	    =   RESOURCE_FILES_NAMES

      def __init__(     self					,      snake_case_					,      snake_case_=None					,      snake_case_=False					,      snake_case_="utf8"					,      snake_case_="[UNK]"					,      snake_case_="[SEP]"					,      snake_case_="[PAD]"					,      snake_case_="[CLS]"					,      snake_case_="[MASK]"					,      snake_case_ = None					,      **snake_case_					,      ):

        """simple docstring"""

        A_  :       List[Any]										=  {} if sp_model_kwargs is None else sp_model_kwargs
        super().__init__(
            do_lower_case=snake_case_					,      unk_token=snake_case_					,      sep_token=snake_case_					,      pad_token=snake_case_					,      cls_token=snake_case_					,      mask_token=snake_case_					,      vocab_file=snake_case_					,      encoding=snake_case_					,      sp_model_kwargs=self.sp_model_kwargs					,      **snake_case_					,      )
        A_  :       List[Any]										=  do_lower_case
        A_  :       Optional[Any]										=  sentencepiece_model_ckpt
        A_  :       int										=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(snake_case_  )

        # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
        if vocab_file is not None:
          A_  :       Optional[int]										=  self.load_vocab(filepath=snake_case_  )
        else:
          A_  :       Tuple										=  {self.sp_model.id_to_piece(snake_case_  ): id for id in range(self.sp_model.get_piece_size()  )}
        A_  :       Optional[Any]										=  {v: k for k, v in self.vocab.items()}

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        if text is None:
          return None

        A_  :       Any										=  self.tokenize(snake_case_  )
        A_						,       A_  :       str										=  '', []

        for i, ch in enumerate(snake_case_  ):
          if ch in self.SP_CHAR_MAPPING:
            A_  :       int										=  self.SP_CHAR_MAPPING.get(snake_case_  )
          else:
            A_  :       Optional[int]										=  unicodedata.normalize('NFKC'					,      snake_case_  )
          if self.is_whitespace(snake_case_  ):
            continue
          normalized_text += ch
          char_mapping.extend([i] * len(snake_case_  )  )

        A_						,       A_						,       A_  :       Dict										=  normalized_text, [], 0

        if self.do_lower_case:
          A_  :       Tuple										=  text.lower()

        for token in split_tokens:
          if token[:1] == "▁":
            A_  :       Union[str, Any]										=  token[1:]
          A_  :       Any										=  text[offset:].index(snake_case_  ) + offset
          A_  :       List[str]										=  start + len(snake_case_  )

          token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
          A_  :       Tuple										=  end
        return token_mapping

      @property
      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        return len(self.vocab  )

      def 					lowerCamelCase_			(     self  ):

        """simple docstring"""

        return dict(self.vocab					,      **self.added_tokens_encoder  )

      def __getstate__(     self  ):

        """simple docstring"""

        A_  :       List[Any]										=  self.__dict__.copy()
        A_  :       Union[str, Any]										=  None
        return state

      def __setstate__(     self					,      snake_case_  ):

        """simple docstring"""

        A_  :       str										=  d

        # for backward compatibility
        if not hasattr(self					,      'sp_model_kwargs'  ):
          A_  :       List[Any]										=  {}

        A_  :       Optional[Any]										=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
        self.sp_model.Load(self.sentencepiece_model_ckpt  )

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        return "".join((self.SP_CHAR_MAPPING.get(snake_case_					,      snake_case_  ) for c in text)  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_=False					,      snake_case_=6_4					,      snake_case_=0.1  ):

        """simple docstring"""

        if self.sp_model_kwargs.get('enable_sampling'  ) is True:
          A_  :       Dict										=  True
        if self.sp_model_kwargs.get('alpha'  ) is not None:
          A_  :       Optional[int]										=  self.sp_model_kwargs.get('alpha'  )
        if self.sp_model_kwargs.get('nbest_size'  ) is not None:
          A_  :       List[str]										=  self.sp_model_kwargs.get('nbest_size'  )

        if not enable_sampling:
          A_  :       Union[str, Any]										=  self.sp_model.EncodeAsPieces(snake_case_  )
        else:
          A_  :       str										=  self.sp_model.SampleEncodeAsPieces(snake_case_					,      snake_case_					,      snake_case_  )
        A_  :       str										=  []
        for pi, piece in enumerate(snake_case_  ):
          if piece == SPIECE_UNDERLINE:
            if not pieces[pi + 1].startswith(snake_case_  ) and pi != 0:
              new_pieces.append(snake_case_  )
              continue
            else:
              continue
          A_  :       int										=  0
          for i, chunk in enumerate(snake_case_  ):
            if chunk == SPIECE_UNDERLINE:
              continue
            if self.is_ch_char(snake_case_  ) or self.is_punct(snake_case_  ):
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]  )
              new_pieces.append(snake_case_  )
              A_  :       Union[str, Any]										=  i + 1
            elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]  )
              A_  :       Tuple										=  i
            elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
              if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]  )
              A_  :       str										=  i
          if len(snake_case_  ) > lst_i:
            new_pieces.append(piece[lst_i:]  )
        return new_pieces

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        A_  :       Any										=  ''.join(snake_case_  ).replace(snake_case_					,      ' '  ).strip()
        return out_string

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        A_  :       Tuple										=  self.convert_ids_to_tokens(snake_case_  )
        A_  :       Tuple										=  ''.join(snake_case_  ).replace(snake_case_					,      ' '  ).strip()
        return out_string

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        return self.vocab.get(snake_case_					,      self.vocab.get(self.unk_token  )  )

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        return self.reverse_vocab.get(snake_case_					,      self.unk_token  )

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_=None  ):

        """simple docstring"""

        if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        A_  :       Any										=  [self.cls_token_id]
        A_  :       Tuple										=  [self.sep_token_id]
        return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_=None  ):

        """simple docstring"""

        if offset_mapping_a is None:
          return [(0, 0)] + offset_mapping_a + [(0, 0)]

        return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_=None					,      snake_case_=False  ):

        """simple docstring"""

        if already_has_special_tokens:
          if token_ids_a is not None:
            raise ValueError(
                'You should not supply a second sequence if the provided sequence of '
                'ids is already formatted with special tokens for the model.'  )
          return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is not None:
          return [1] + ([0] * len(snake_case_  )) + [1, 1] + ([0] * len(snake_case_  )) + [1]
        return [1] + ([0] * len(snake_case_  )) + [1]

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_ = None  ):

        """simple docstring"""

        if token_ids_a is None:
          # [CLS] X [SEP]
          return (len(snake_case_  ) + 2) * [0]

        # [CLS] A [SEP] [SEP] B [SEP]
        return [0] * (len(snake_case_  ) + 1) + [1] * (len(snake_case_  ) + 3)

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        if "\u4e00" <= char <= "\u9fff":
          return True
        return False

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        if ("a" <= char <= "z") or ("A" <= char <= "Z"):
          return True
        return False

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        if char in ",;:.?!~，；：。？！《》【】":
          return True
        return False

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        if char == " " or char == "\t" or char == "\n" or char == "\r":
          return True
        if len(snake_case_  ) == 1:
          A_  :       Any										=  unicodedata.category(snake_case_  )
          if cat == "Zs":
            return True
        return False

      def 					lowerCamelCase_			(     self					,      snake_case_  ):

        """simple docstring"""

        A_  :       Union[str, Any]										=  {}
        with io.open(snake_case_					,      'r'					,      encoding='utf-8'  ) as f:
          for index, line in enumerate(snake_case_  ):
            A_  :       Optional[int]										=  line.rstrip('\n'  )
            A_  :       Optional[int]										=  int(snake_case_  )

        return token_to_idx

      def 					lowerCamelCase_			(     self					,      snake_case_					,      snake_case_ = None  ):

        """simple docstring"""

        A_  :       List[str]										=  0
        if os.path.isdir(snake_case_  ):
          A_  :       Any										=  os.path.join(
              snake_case_					,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
        else:
          A_  :       List[Any]										=  (filename_prefix + '-' if filename_prefix else '') + save_directory
        with open(snake_case_					,      'w'					,      encoding='utf-8'  ) as writer:
          for token, token_index in sorted(self.vocab.items()					,      key=lambda snake_case_  : kv[1]  ):
            if index != token_index:
              logger.warning(
                  F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                  ' Please check that the vocabulary is not corrupted!'  )
              A_  :       Optional[Any]										=  token_index
            writer.write(token + '\n'  )
            index += 1

        A_  :       List[Any]										=  os.path.join(snake_case_					,      'sentencepiece.bpe.model'  )
        with open(snake_case_					,      'wb'  ) as fi:
          A_  :       Dict										=  self.sp_model.serialized_model_proto()
          fi.write(snake_case_  )

        return (vocab_file,)






"""simple docstring"""




import os





def 						UpperCAmelCase__		(     ):






  """simple docstring"""





  A_  :       Any										=  os.path.join(os.path.dirname(_UpperCAmelCase					) ,					'num.txt'					)
  with open(_UpperCAmelCase					) as file_hand:
    return str(sum(int(_UpperCAmelCase					) for line in file_hand					)					)[:10]


if __name__ == "__main__":
   print(solution())