





"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__						= logging.get_logger(__name__)


lowerCamelCase__						= "▁"

lowerCamelCase__						= {"vocab_file": "spiece.model"}

lowerCamelCase__						= {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

lowerCamelCase__						= {
    "google/reformer-crime-and-punishment": 52_4288,
}



class   __SCREAMING_SNAKE_CASE     (    _UpperCamelCase ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__				:Optional[Any]						  =       VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__				:int						  =       PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__				:Any						  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__				:Tuple						  =       ["input_ids", "attention_mask"]

  def __init__(		self :							Optional[int]       , __a :							List[Any]       , __a :							Any="</s>"       , __a :							Any="<unk>"       , __a :							List[Any]=[]       , __a :							Optional[Dict[str, Any]] = None       , **__a :							Optional[Any]       , )		->					None:
        _UpperCamelCase   :						Any    =							{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            eos_token=__a       , unk_token=__a       , additional_special_tokens=__a       , sp_model_kwargs=self.sp_model_kwargs       , **__a       , )

        _UpperCamelCase   :						List[str]    =							vocab_file
        _UpperCamelCase   :						Optional[int]    =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(__a						)

  @property
  def  __SCREAMING_SNAKE_CASE					(		self :							Optional[Any]						)		->					Any:
        return self.sp_model.get_piece_size()

  def  __SCREAMING_SNAKE_CASE					(		self :							Optional[int]						)		->					Dict[str, int]:
        _UpperCamelCase   :						int    =							{self.convert_ids_to_tokens(__a						): i for i in range(self.vocab_size						)}
        vocab.update(self.added_tokens_encoder						)
        return vocab

  def __getstate__(		self :							Tuple						)		->					List[str]:
        _UpperCamelCase   :						Any    =							self.__dict__.copy()
        _UpperCamelCase   :						Union[str, Any]    =							None
        return state

  def __setstate__(		self :							Optional[int]       , __a :							Tuple						)		->					Dict:
        _UpperCamelCase   :						List[Any]    =							d

        # for backward compatibility
        if not hasattr(self       , "sp_model_kwargs"						):
              _UpperCamelCase   :						Optional[Any]    =							{}

        _UpperCamelCase   :						Optional[int]    =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(self.vocab_file						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Tuple       , __a :							str						)		->					List[str]:
        return self.sp_model.encode(__a       , out_type=__a						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Union[str, Any]       , __a :							List[Any]						)		->					Optional[int]:
        return self.sp_model.piece_to_id(__a						)

  def  __SCREAMING_SNAKE_CASE					(		self :							List[Any]       , __a :							Union[str, Any]						)		->					List[str]:
        if index < self.sp_model.get_piece_size():
              _UpperCamelCase   :						List[str]    =							self.sp_model.IdToPiece(__a						)
        return token

  def  __SCREAMING_SNAKE_CASE					(		self :							List[Any]       , __a :							Dict						)		->					str:
        _UpperCamelCase   :						List[Any]    =							[]
        _UpperCamelCase   :						List[str]    =							""
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(__a						) + token
                    _UpperCamelCase   :						Union[str, Any]    =							[]
              else:
                    current_sub_tokens.append(__a						)
        out_string += self.sp_model.decode(__a						)
        return out_string.strip()


  def  __SCREAMING_SNAKE_CASE					(		self :							Dict       , __a :							str       , __a :							Optional[str] = None						)		->					Tuple[str]:
        if not os.path.isdir(__a						):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
              return
        _UpperCamelCase   :						Optional[Any]    =							os.path.join(
            __a       , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

        if os.path.abspath(self.vocab_file						) != os.path.abspath(__a						) and os.path.isfile(self.vocab_file						):
              copyfile(self.vocab_file       , __a						)
        elif not os.path.isfile(self.vocab_file						):
              with open(__a       , "wb"						) as fi:
                    _UpperCamelCase   :						str    =							self.sp_model.serialized_model_proto()
                    fi.write(__a						)

        return (out_vocab_file,)







"""simple docstring"""


import builtins
import sys

from ...utils.imports import _is_package_available
from . import cursor, input
from .helpers import Direction, clear_line, forceWrite, linebreak, move_cursor, reset_cursor, writeColor
from .keymap import KEYMAP


lowerCamelCase__						= False
try:
  lowerCamelCase__						= _is_package_available("google.colab")
except ModuleNotFoundError:
  pass



@input.register
class   __SCREAMING_SNAKE_CASE     :







  '''simple docstring'''


  def __init__(		self :							Union[str, Any]       , __a :							str = None       , __a :							list = []						)		->					Union[str, Any]:
        _UpperCamelCase   :						Any    =							0
        _UpperCamelCase   :						Any    =							choices
        _UpperCamelCase   :						Tuple    =							prompt
        if sys.platform == "win32":
              _UpperCamelCase   :						str    =							"*"
        else:
              _UpperCamelCase   :						Dict    =							"➔ "

  def  __SCREAMING_SNAKE_CASE					(		self :							int       , __a :							Tuple       , __a :							str = ""						)		->					Dict:
        if sys.platform != "win32":
              writeColor(self.choices[index]       , 32       , __a						)
        else:
              forceWrite(self.choices[index]       , __a						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Tuple       , __a :							int						)		->					Dict:
        if index == self.position:
              forceWrite(F''' {self.arrow_char} '''						)
              self.write_choice(__a						)
        else:
              forceWrite(F'''    {self.choices[index]}'''						)
        reset_cursor()

  def  __SCREAMING_SNAKE_CASE					(		self :							List[str]       , __a :							Direction       , __a :							int = 1						)		->					List[str]:
        _UpperCamelCase   :						Tuple    =							self.position
        if direction == Direction.DOWN:
              if self.position + 1 >= len(self.choices						):
                    return
              self.position += num_spaces
        else:
              if self.position - 1 < 0:
                    return
              self.position -= num_spaces
        clear_line()
        self.print_choice(__a						)
        move_cursor(__a       , direction.name						)
        self.print_choice(self.position						)

  @input.mark(KEYMAP["up"]						)
  def  __SCREAMING_SNAKE_CASE					(		self :							Union[str, Any]						)		->					Any:
        self.move_direction(Direction.UP						)

  @input.mark(KEYMAP["down"]						)
  def  __SCREAMING_SNAKE_CASE					(		self :							Dict						)		->					List[Any]:
        self.move_direction(Direction.DOWN						)

  @input.mark(KEYMAP["newline"]						)
  def  __SCREAMING_SNAKE_CASE					(		self :							int						)		->					str:
        move_cursor(len(self.choices						) - self.position       , "DOWN"						)
        return self.position

  @input.mark(KEYMAP["interrupt"]						)
  def  __SCREAMING_SNAKE_CASE					(		self :							Optional[Any]						)		->					Tuple:
        move_cursor(len(self.choices						) - self.position       , "DOWN"						)
        raise KeyboardInterrupt

  @input.mark_multiple(*[KEYMAP[str(__a						)] for number in range(10						)]						)
  def  __SCREAMING_SNAKE_CASE					(		self :							Optional[int]						)		->					List[str]:
        _UpperCamelCase   :						Union[str, Any]    =							int(chr(self.current_selection						)						)
        _UpperCamelCase   :						List[str]    =							index - self.position
        if index == self.position:
              return
        if index < len(self.choices						):
              if self.position > index:
                    self.move_direction(Direction.UP       , -movement						)
              elif self.position < index:
                    self.move_direction(Direction.DOWN       , __a						)
              else:
                    return
        else:
              return


  def  __SCREAMING_SNAKE_CASE					(		self :							List[Any]       , __a :							int = 0						)		->					List[Any]:
        if self.prompt:
              linebreak()
              forceWrite(self.prompt       , "\n"						)
              if in_colab:
                    forceWrite("Please input a choice index (starting from 0), and press enter"       , "\n"						)
              else:
                    forceWrite("Please select a choice using the arrow or number keys, and selecting with enter"       , "\n"						)
        _UpperCamelCase   :						int    =							default_choice
        for i in range(len(self.choices						)						):
              self.print_choice(__a						)
              forceWrite("\n"						)
        move_cursor(len(self.choices						) - self.position       , "UP"						)
        with cursor.hide():
              while True:
                    if in_colab:
                          try:
                                _UpperCamelCase   :						Union[str, Any]    =							int(builtins.input()						)
                          except ValueError:
                                _UpperCamelCase   :						int    =							default_choice
                    else:
                          _UpperCamelCase   :						Any    =							self.handle_input()
                    if choice is not None:
                          reset_cursor()
                          for _ in range(len(self.choices						) + 1						):
                                move_cursor(1       , "UP"						)
                                clear_line()
                          self.write_choice(__a       , "\n"						)
                          return choice
