





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_     =     logging.get_logger(__name__)
lowerCAmelCase_     =     {"vocab_file": "spiece.model"}

lowerCAmelCase_     =     {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

lowerCAmelCase_     =     {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

lowerCAmelCase_     =     "▁"



class 							_lowerCAmelCase     (      UpperCAmelCase_				):


  '''simple docstring'''
  a_				:							Tuple          =VOCAB_FILES_NAMES
  a_				:							Dict          =PRETRAINED_VOCAB_FILES_MAP
  a_				:							str          =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







  def __init__(   self   :       List[str]   ,					UpperCamelCase   :       Tuple   ,					UpperCamelCase   :       Optional[int]=True   ,					UpperCamelCase   :       List[Any]=True   ,					UpperCamelCase   :       str=False   ,					UpperCamelCase   :       str="[CLS]"   ,					UpperCamelCase   :       Optional[int]="[SEP]"   ,					UpperCamelCase   :       int="<unk>"   ,					UpperCamelCase   :       str="[SEP]"   ,					UpperCamelCase   :       Dict="<pad>"   ,					UpperCamelCase   :       int="[CLS]"   ,					UpperCamelCase   :       Tuple="[MASK]"   ,					UpperCamelCase   :       Optional[Dict[str, Any]] = None   ,					**UpperCamelCase   :       List[str]   ,					):


    '''simple docstring'''





    _snake_case :					Dict								=						(
        AddedToken(UpperCamelCase   ,					lstrip=UpperCamelCase   ,					rstrip=UpperCamelCase   ,					normalized=UpperCamelCase				)
        if isinstance(UpperCamelCase   ,					UpperCamelCase				)
        else mask_token
    )

    _snake_case :					List[str]								=						{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=UpperCamelCase   ,					remove_space=UpperCamelCase   ,					keep_accents=UpperCamelCase   ,					bos_token=UpperCamelCase   ,					eos_token=UpperCamelCase   ,					unk_token=UpperCamelCase   ,					sep_token=UpperCamelCase   ,					pad_token=UpperCamelCase   ,					cls_token=UpperCamelCase   ,					mask_token=UpperCamelCase   ,					sp_model_kwargs=self.sp_model_kwargs   ,					**UpperCamelCase   ,					)

    _snake_case :					List[str]								=						do_lower_case
    _snake_case :					str								=						remove_space
    _snake_case :					Tuple								=						keep_accents
    _snake_case :					Dict								=						vocab_file

    _snake_case :					List[Any]								=						spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(UpperCamelCase				)







  @property
  def UpperCamelCase_			(   self   :       Any				):


    '''simple docstring'''





    return len(self.sp_model				)







  def UpperCamelCase_			(   self   :       Optional[int]				):


    '''simple docstring'''





    _snake_case :					Optional[int]								=						{self.convert_ids_to_tokens(UpperCamelCase				): i for i in range(self.vocab_size				)}
    vocab.update(self.added_tokens_encoder				)
    return vocab







  def __getstate__(   self   :       int				):


    '''simple docstring'''





    _snake_case :					Union[str, Any]								=						self.__dict__.copy()
    _snake_case :					List[str]								=						None
    return state







  def __setstate__(   self   :       int   ,					UpperCamelCase   :       str				):


    '''simple docstring'''





    _snake_case :					str								=						d

    # for backward compatibility
    if not hasattr(self   ,					'sp_model_kwargs'				):
      _snake_case :					Dict								=						{}

    _snake_case :					Any								=						spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(self.vocab_file				)







  def UpperCamelCase_			(   self   :       Dict   ,					UpperCamelCase   :       int				):


    '''simple docstring'''





    if self.remove_space:
      _snake_case :					str								=						' '.join(inputs.strip().split()				)
    else:
      _snake_case :					Union[str, Any]								=						inputs
    _snake_case :					str								=						outputs.replace('``'   ,					'"'				).replace('\'\''   ,					'"'				)

    if not self.keep_accents:
      _snake_case :					Dict								=						unicodedata.normalize('NFKD'   ,					UpperCamelCase				)
      _snake_case :					Any								=						''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase				)]				)
    if self.do_lower_case:
      _snake_case :					Union[str, Any]								=						outputs.lower()

    return outputs







  def UpperCamelCase_			(   self   :       Any   ,					UpperCamelCase   :       str				):


    '''simple docstring'''





    _snake_case :					List[Any]								=						self.preprocess_text(UpperCamelCase				)
    _snake_case :					List[str]								=						self.sp_model.encode(UpperCamelCase   ,					out_type=UpperCamelCase				)
    _snake_case :					str								=						[]
    for piece in pieces:
      if len(UpperCamelCase				) > 1 and piece[-1] == str(','				) and piece[-2].isdigit():
        _snake_case :					Optional[Any]								=						self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase   ,					''				)				)
        if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
          if len(cur_pieces[0]				) == 1:
            _snake_case :					Tuple								=						cur_pieces[1:]
          else:
            _snake_case :					Dict								=						cur_pieces[0][1:]
        cur_pieces.append(piece[-1]				)
        new_pieces.extend(UpperCamelCase				)
      else:
        new_pieces.append(UpperCamelCase				)

    return new_pieces







  def UpperCamelCase_			(   self   :       Any   ,					UpperCamelCase   :       int				):


    '''simple docstring'''





    return self.sp_model.PieceToId(UpperCamelCase				)







  def UpperCamelCase_			(   self   :       List[str]   ,					UpperCamelCase   :       Any				):


    '''simple docstring'''





    return self.sp_model.IdToPiece(UpperCamelCase				)







  def UpperCamelCase_			(   self   :       Tuple   ,					UpperCamelCase   :       Optional[Any]				):


    '''simple docstring'''





    _snake_case :					Dict								=						[]
    _snake_case :					Dict								=						''
    _snake_case :					List[str]								=						False
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        if not prev_is_special:
          out_string += " "
        out_string += self.sp_model.decode(UpperCamelCase				) + token
        _snake_case :					int								=						True
        _snake_case :					Optional[Any]								=						[]
      else:
        current_sub_tokens.append(UpperCamelCase				)
        _snake_case :					List[str]								=						False
    out_string += self.sp_model.decode(UpperCamelCase				)
    return out_string.strip()







  def UpperCamelCase_			(   self   :       Union[str, Any]   ,					UpperCamelCase   :       List[int]   ,					UpperCamelCase   :       Optional[List[int]] = None				):


    '''simple docstring'''





    _snake_case :					Any								=						[self.sep_token_id]
    _snake_case :					int								=						[self.cls_token_id]
    if token_ids_a is None:
      return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep







  def UpperCamelCase_			(   self   :       int   ,					UpperCamelCase   :       List[int]   ,					UpperCamelCase   :       Optional[List[int]] = None   ,					UpperCamelCase   :       bool = False				):


    '''simple docstring'''





    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=UpperCamelCase   ,					token_ids_a=UpperCamelCase   ,					already_has_special_tokens=UpperCamelCase				)

    if token_ids_a is not None:
      return [1] + ([0] * len(UpperCamelCase				)) + [1] + ([0] * len(UpperCamelCase				)) + [1]
    return [1] + ([0] * len(UpperCamelCase				)) + [1]







  def UpperCamelCase_			(   self   :       Union[str, Any]   ,					UpperCamelCase   :       List[int]   ,					UpperCamelCase   :       Optional[List[int]] = None				):


    '''simple docstring'''





    _snake_case :					Union[str, Any]								=						[self.sep_token_id]
    _snake_case :					str								=						[self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







  def UpperCamelCase_			(   self   :       Dict   ,					UpperCamelCase   :       str   ,					UpperCamelCase   :       Optional[str] = None				):


    '''simple docstring'''





    if not os.path.isdir(UpperCamelCase				):
      logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
      return
    _snake_case :					List[Any]								=						os.path.join(
        UpperCamelCase   ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)

    if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase				) and os.path.isfile(self.vocab_file				):
      copyfile(self.vocab_file   ,					UpperCamelCase				)
    elif not os.path.isfile(self.vocab_file				):
      with open(UpperCamelCase   ,					'wb'				) as fi:
        _snake_case :					Tuple								=						self.sp_model.serialized_model_proto()
        fi.write(UpperCamelCase				)

    return (out_vocab_file,)









from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching

def 	_lowerCAmelCase (    ):
      import os as original_os
      from os import path as original_path
      from os import rename as original_rename
      from os.path import dirname as original_dirname
      from os.path import join as original_join

      assert _test_patching.os is original_os
      assert _test_patching.path is original_path
      assert _test_patching.join is original_join

      assert _test_patching.renamed_os is original_os
      assert _test_patching.renamed_path is original_path
      assert _test_patching.renamed_join is original_join

      lowercase__  =	'__test_patch_submodule_mock__'
      with patch_submodule(_test_patching    ,	'os.path.join'    ,	A__						):
            # Every way to access os.path.join must be patched, and the rest must stay untouched

            # check os.path.join
            assert isinstance(_test_patching.os    ,	_PatchedModuleObj						)
            assert isinstance(_test_patching.os.path    ,	_PatchedModuleObj						)
            assert _test_patching.os.path.join is mock

            # check path.join
            assert isinstance(_test_patching.path    ,	_PatchedModuleObj						)
            assert _test_patching.path.join is mock

            # check join
            assert _test_patching.join is mock

            # check that the other attributes are untouched
            assert _test_patching.os.rename is original_rename
            assert _test_patching.path.dirname is original_dirname
            assert _test_patching.os.path.dirname is original_dirname

            # Even renamed modules or objects must be patched

            # check renamed_os.path.join
            assert isinstance(_test_patching.renamed_os    ,	_PatchedModuleObj						)
            assert isinstance(_test_patching.renamed_os.path    ,	_PatchedModuleObj						)
            assert _test_patching.renamed_os.path.join is mock

            # check renamed_path.join
            assert isinstance(_test_patching.renamed_path    ,	_PatchedModuleObj						)
            assert _test_patching.renamed_path.join is mock

            # check renamed_join
            assert _test_patching.renamed_join is mock

            # check that the other attributes are untouched
            assert _test_patching.renamed_os.rename is original_rename
            assert _test_patching.renamed_path.dirname is original_dirname
            assert _test_patching.renamed_os.path.dirname is original_dirname

      # check that everthing is back to normal when the patch is over

      assert _test_patching.os is original_os
      assert _test_patching.path is original_path
      assert _test_patching.join is original_join

      assert _test_patching.renamed_os is original_os
      assert _test_patching.renamed_path is original_path
      assert _test_patching.renamed_join is original_join

def 	_lowerCAmelCase (    ):
      assert _test_patching.open is open

      lowercase__  =	'__test_patch_submodule_builtin_mock__'
      # _test_patching has "open" in its globals
      assert _test_patching.open is open
      with patch_submodule(_test_patching    ,	'open'    ,	A__						):
            assert _test_patching.open is mock

      # check that everthing is back to normal when the patch is over

      assert _test_patching.open is open

def 	_lowerCAmelCase (    ):
      # pandas.read_csv is not present in _test_patching
      lowercase__  =	'__test_patch_submodule_missing_mock__'
      with patch_submodule(_test_patching    ,	'pandas.read_csv'    ,	A__						):
            pass

def 	_lowerCAmelCase (    ):
      # builtin should always be mocked even if they're not in the globals
      # in case they're loaded at one point
      lowercase__  =	'__test_patch_submodule_missing_builtin_mock__'
      # _test_patching doesn't have "len" in its globals
      assert getattr(_test_patching    ,	'len'    ,	A__						) is None
      with patch_submodule(_test_patching    ,	'len'    ,	A__						):
            assert _test_patching.len is mock
      assert _test_patching.len is len

def 	_lowerCAmelCase (    ):
      lowercase__  =	'__test_patch_submodule_start_and_stop_mock__'
      lowercase__  =	patch_submodule(_test_patching    ,	'open'    ,	A__						)
      assert _test_patching.open is open
      patch.start()
      assert _test_patching.open is mock
      patch.stop()
      assert _test_patching.open is open

def 	_lowerCAmelCase (    ):
      from os import rename as original_rename
      from os.path import dirname as original_dirname
      from os.path import join as original_join

      lowercase__  =	'__test_patch_submodule_successive_join__'
      lowercase__  =	'__test_patch_submodule_successive_dirname__'
      lowercase__  =	'__test_patch_submodule_successive_rename__'
      assert _test_patching.os.path.join is original_join
      assert _test_patching.os.path.dirname is original_dirname
      assert _test_patching.os.rename is original_rename

      with patch_submodule(_test_patching    ,	'os.path.join'    ,	A__						):
            with patch_submodule(_test_patching    ,	'os.rename'    ,	A__						):
                  with patch_submodule(_test_patching    ,	'os.path.dirname'    ,	A__						):
                        assert _test_patching.os.path.join is mock_join
                        assert _test_patching.os.path.dirname is mock_dirname
                        assert _test_patching.os.rename is mock_rename

    # try another order
      with patch_submodule(_test_patching    ,	'os.rename'    ,	A__						):
            with patch_submodule(_test_patching    ,	'os.path.join'    ,	A__						):
                  with patch_submodule(_test_patching    ,	'os.path.dirname'    ,	A__						):
                        assert _test_patching.os.path.join is mock_join
                        assert _test_patching.os.path.dirname is mock_dirname
                        assert _test_patching.os.rename is mock_rename

      assert _test_patching.os.path.join is original_join
      assert _test_patching.os.path.dirname is original_dirname
      assert _test_patching.os.rename is original_rename

def 	_lowerCAmelCase (    ):
      lowercase__  =	'__test_patch_submodule_doesnt_exist_mock__'
      with patch_submodule(_test_patching    ,	'__module_that_doesn_exist__.__attribute_that_doesn_exist__'    ,	A__						):
            pass
      with patch_submodule(_test_patching    ,	'os.__attribute_that_doesn_exist__'    ,	A__						):
            pass
