

'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase					:			Union[str, Any]       =	logging.get_logger(__name__)

UpperCAmelCase					:			List[str]       =	{'vocab_file': 'spm_char.model'}

UpperCAmelCase					:			List[Any]       =	{
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

UpperCAmelCase					:			Dict       =	{
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 lowerCAmelCase__       = VOCAB_FILES_NAMES
 lowerCAmelCase__       = PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       = ["input_ids", "attention_mask"]
 def __init__(						self   :			int   ,				__SCREAMING_SNAKE_CASE   :			str   ,				__SCREAMING_SNAKE_CASE   :			Dict="<s>"   ,				__SCREAMING_SNAKE_CASE   :			str="</s>"   ,				__SCREAMING_SNAKE_CASE   :			Tuple="<unk>"   ,				__SCREAMING_SNAKE_CASE   :			int="<pad>"   ,				__SCREAMING_SNAKE_CASE   :			Optional[Dict[str, Any]] = None   ,				**__SCREAMING_SNAKE_CASE   :			List[str]   ,				)    ->					None:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=__SCREAMING_SNAKE_CASE   ,				eos_token=__SCREAMING_SNAKE_CASE   ,				unk_token=__SCREAMING_SNAKE_CASE   ,				pad_token=__SCREAMING_SNAKE_CASE   ,				sp_model_kwargs=self.sp_model_kwargs   ,				**__SCREAMING_SNAKE_CASE   ,				)

      __SCREAMING_SNAKE_CASE      = vocab_file

      __SCREAMING_SNAKE_CASE      = spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(__SCREAMING_SNAKE_CASE     )
 @property
 def  UpperCAmelCase__						(						self   :			Union[str, Any]     )    ->					Dict:


      """simple docstring"""




      return self.sp_model.get_piece_size()
 def  UpperCAmelCase__						(						self   :			List[Any]     )    ->					Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab
 def __getstate__(						self   :			int     )    ->					Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.__dict__.copy()
      __SCREAMING_SNAKE_CASE      = None
      return state
 def __setstate__(						self   :			int   ,				__SCREAMING_SNAKE_CASE   :			Dict     )    ->					Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = d

      # for backward compatibility
      if not hasattr(self   ,				"""sp_model_kwargs"""     ):
           __SCREAMING_SNAKE_CASE      = {}

      __SCREAMING_SNAKE_CASE      = spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )
 def  UpperCAmelCase__						(						self   :			str   ,				__SCREAMING_SNAKE_CASE   :			str     )    ->					List[str]:


      """simple docstring"""




      return self.sp_model.encode(__SCREAMING_SNAKE_CASE   ,				out_type=__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Any   ,				__SCREAMING_SNAKE_CASE   :			Optional[int]     )    ->					Tuple:


      """simple docstring"""




      return self.sp_model.piece_to_id(__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Tuple   ,				__SCREAMING_SNAKE_CASE   :			List[str]     )    ->					int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE     )
      return token
 def  UpperCAmelCase__						(						self   :			Any   ,				__SCREAMING_SNAKE_CASE   :			List[str]     )    ->					Any:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = []
      __SCREAMING_SNAKE_CASE      = """"""
      for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
                out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE     ) + token
                __SCREAMING_SNAKE_CASE      = []
           else:
                current_sub_tokens.append(__SCREAMING_SNAKE_CASE     )
      out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE     )
      return out_string.strip()
 def  UpperCAmelCase__						(						self   :			Dict   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]   ,				__SCREAMING_SNAKE_CASE   :			List[Any]=None     )    ->					List[int]:


      """simple docstring"""




      if token_ids_a is None:
           return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]
 def  UpperCAmelCase__						(						self   :			Tuple   ,				__SCREAMING_SNAKE_CASE   :			List[int]   ,				__SCREAMING_SNAKE_CASE   :			Optional[List[int]] = None   ,				__SCREAMING_SNAKE_CASE   :			bool = False     )    ->					List[int]:


      """simple docstring"""




      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__SCREAMING_SNAKE_CASE   ,				token_ids_a=__SCREAMING_SNAKE_CASE   ,				already_has_special_tokens=__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = [1]
      if token_ids_a is None:
           return ([0] * len(__SCREAMING_SNAKE_CASE     )) + suffix_ones
      return ([0] * len(__SCREAMING_SNAKE_CASE     )) + ([0] * len(__SCREAMING_SNAKE_CASE     )) + suffix_ones


 def  UpperCAmelCase__						(						self   :			Optional[Any]   ,				__SCREAMING_SNAKE_CASE   :			str   ,				__SCREAMING_SNAKE_CASE   :			Optional[str] = None     )    ->					Tuple[str]:


      """simple docstring"""




      if not os.path.isdir(__SCREAMING_SNAKE_CASE     ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
           return
      __SCREAMING_SNAKE_CASE      = os.path.join(
          __SCREAMING_SNAKE_CASE   ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(__SCREAMING_SNAKE_CASE     ) and os.path.isfile(self.vocab_file     ):
           copyfile(self.vocab_file   ,				__SCREAMING_SNAKE_CASE     )
      elif not os.path.isfile(self.vocab_file     ):
           with open(__SCREAMING_SNAKE_CASE   ,				"""wb"""     ) as fi:
                __SCREAMING_SNAKE_CASE      = self.sp_model.serialized_model_proto()
                fi.write(__SCREAMING_SNAKE_CASE     )

      return (out_vocab_file,)









'''simple docstring'''



UpperCAmelCase					:			Tuple       =	range(2, 2_0 + 1)
UpperCAmelCase					:			int       =	[1_0**k for k in range(ks[-1] + 1)]
UpperCAmelCase					:			dict[int, dict[int, list[list[int]]]]       =	{}







def   a__			(    a__ ,       a__ ,       a__ ,       a__		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = sum(a_i[j] for j in range(a__ ,       len(a__		)		)		)
     __SCREAMING_SNAKE_CASE      = sum(a_i[j] * base[j] for j in range(min(len(a__		) ,       a__		)		)		)

     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = 0, 0
     __SCREAMING_SNAKE_CASE      = n - i

     __SCREAMING_SNAKE_CASE      = memo.get(a__		)

     if sub_memo is not None:
          __SCREAMING_SNAKE_CASE      = sub_memo.get(a__		)

          if jumps is not None and len(a__		) > 0:
               # find and make the largest jump without going over
               __SCREAMING_SNAKE_CASE      = -1
               for _k in range(len(a__		) - 1 ,       -1 ,       -1		):
                    if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
                         __SCREAMING_SNAKE_CASE      = _k
                         break

               if max_jump >= 0:
                    __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = jumps[max_jump]
                    # since the difference between jumps is cached, add c
                    __SCREAMING_SNAKE_CASE      = diff + c
                    for j in range(min(a__ ,       len(a__		)		)		):
                         __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
                    if new_c > 0:
                         add(a__ ,       a__ ,       a__		)

          else:
               __SCREAMING_SNAKE_CASE      = []
     else:
          __SCREAMING_SNAKE_CASE      = {c: []}
          __SCREAMING_SNAKE_CASE      = sub_memo

     if dn >= max_dn or c + diff >= base[k]:
          return diff, dn

     if k > ks[0]:
          while True:
               # keep doing smaller jumps
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = next_term(a__ ,       k - 1 ,       i + dn ,       a__		)
               diff += _diff
               dn += terms_jumped

               if dn >= max_dn or c + diff >= base[k]:
                    break
     else:
          # would be too small a jump, just compute sequential terms instead
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = compute(a__ ,       a__ ,       i + dn ,       a__		)
          diff += _diff
          dn += terms_jumped

     __SCREAMING_SNAKE_CASE      = sub_memo[c]

     # keep jumps sorted by # of terms skipped
     __SCREAMING_SNAKE_CASE      = 0
     while j < len(a__		):
          if jumps[j][1] > dn:
               break
          j += 1

     # cache the jump for this value digitsum(b) and c
     sub_memo[c].insert(a__ ,       (diff, dn, k)		)
     return (diff, dn)







def   a__			(    a__ ,       a__ ,       a__ ,       a__		):




     """simple docstring"""


     if i >= n:
          return 0, i
     if k > len(a__		):
          a_i.extend([0 for _ in range(k - len(a__		)		)]		)

     # note: a_i -> b * 10^k + c
     # ds_b -> digitsum(b)
     # ds_c -> digitsum(c)
     __SCREAMING_SNAKE_CASE      = i
     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = 0, 0, 0
     for j in range(len(a__		)		):
          if j >= k:
               ds_b += a_i[j]
          else:
               ds_c += a_i[j]

     while i < n:
          i += 1
          __SCREAMING_SNAKE_CASE      = ds_c + ds_b
          diff += addend
          __SCREAMING_SNAKE_CASE      = 0
          for j in range(a__		):
               __SCREAMING_SNAKE_CASE      = a_i[j] + addend
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)

               ds_c += a_i[j]

          if addend > 0:
               break

     if addend > 0:
          add(a__ ,       a__ ,       a__		)
     return diff, i - start_i







def   a__			(    a__ ,       a__ ,       a__		):




     """simple docstring"""


     for j in range(a__ ,       len(a__		)		):
          __SCREAMING_SNAKE_CASE      = digits[j] + addend
          if s >= 10:
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
               __SCREAMING_SNAKE_CASE      = addend // 10 + quotient
          else:
               __SCREAMING_SNAKE_CASE      = s
               __SCREAMING_SNAKE_CASE      = addend // 10

          if addend == 0:
               break

     while addend > 0:
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
          digits.append(a__		)







def   a__			(    a__ = 10**15		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = [1]
     __SCREAMING_SNAKE_CASE      = 1
     __SCREAMING_SNAKE_CASE      = 0
     while True:
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = next_term(a__ ,       20 ,       i + dn ,       a__		)
          dn += terms_jumped
          if dn == n - i:
               break

     __SCREAMING_SNAKE_CASE      = 0
     for j in range(len(a__		)		):
          a_n += digits[j] * 10**j
     return a_n


if __name__ == "__main__":
  print(f"""{solution() = }""")






