






"""simple docstring"""



import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
    import torch


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


_UpperCamelCase		:  str						=		logging.get_logger(__name__)
_UpperCamelCase		:  Optional[Any]						=		{'vocab_file': 'spiece.model'}

_UpperCamelCase		:  Tuple						=		{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

_UpperCamelCase		:  int						=		{
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}

class        a     (   a_	):
    UpperCAmelCase_ :     Optional[Any]	     =VOCAB_FILES_NAMES
    UpperCAmelCase_ :     Tuple	     =PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_ :     int	     =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_ :     List[str]	     =["input_ids", "attention_mask"]







    def __init__(			self							,		_lowerCamelCase							,		_lowerCamelCase=False							,		_lowerCamelCase=False							,		_lowerCamelCase=False							,		_lowerCamelCase=None							,		_lowerCamelCase=None							,		_lowerCamelCase=None							,		_lowerCamelCase=None							,		_lowerCamelCase = None							,		**_lowerCamelCase							,		):
      lowercase							    =       {} if sp_model_kwargs is None else sp_model_kwargs

      lowercase							    =       kwargs.get('name_or_path'       )
      if name_or_path is None:
        logger.warning(
            'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
            ' you are testing the model, this can safely be ignored'       )
        lowercase							    =       "None"

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      lowercase							    =       "<|endoftext|>" if eos_token is None else eos_token
      lowercase							    =       "<unk>" if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
        lowercase							    =       unk_token if pad_token is None else pad_token
        lowercase							    =       eos_token if bos_token is None else bos_token
      else:
        lowercase							    =       "<pad>" if pad_token is None else pad_token
        lowercase							    =       "<s>" if bos_token is None else bos_token

      super().__init__(
          do_lower_case=__A							,		remove_space=__A							,		keep_accents=__A							,		bos_token=__A							,		eos_token=__A							,		unk_token=__A							,		pad_token=__A							,		sp_model_kwargs=self.sp_model_kwargs							,		**__A							,		)

      lowercase							    =       do_lower_case
      lowercase							    =       remove_space
      lowercase							    =       keep_accents
      lowercase							    =       vocab_file

      lowercase							    =       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(__A       )

      # Used for whitespace normalization in input texts
      # fmt : off
      lowercase							    =       {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      lowercase							    =       re.compile(
          F'[{"".join(map(__A							,		list(range(0							,		9       )       ) + list(range(1_1							,		3_2       )       ) + list(range(1_2_7							,		1_6_0       )       ) + [1_6_0, 1_7_3, 8_2_0_3]       )       )}]'       )







    def __getstate__(			self       ):
      lowercase							    =       self.__dict__.copy()
      lowercase							    =       None
      return state







    def __setstate__(			self							,		_lowerCamelCase       ):
      lowercase							    =       d

      # for backward compatibility
      if not hasattr(self							,		'sp_model_kwargs'       ):
        lowercase							    =       {}

      lowercase							    =       spm.SentencePieceProcessor(**self.sp_model_kwargs       )
      self.sp_model.Load(self.vocab_file       )







    @property
    # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
    def 			UpperCamelCase_      (			self       ):
      return len(self.sp_model       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      lowercase							    =       self.non_printing_characters_re.sub(''							,		__A       )

      # Normalize whitespaces
      lowercase							    =       "".join([char if char not in self.whitespaces else ' ' for char in text]       )

      # NFC Unicode normalization
      lowercase							    =       unicodedata.normalize('NFC'							,		__A       )
      return text







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		**_lowerCamelCase       ):
      lowercase							    =       self.preprocess_text(__A       )
      return self.sp_model.encode(__A							,		out_type=__A       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      return self.sp_model.PieceToId(__A       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      return self.sp_model.IdToPiece(__A       )







    @staticmethod
    def 			UpperCamelCase_      (			_lowerCamelCase       ):
      return out_string







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      lowercase							    =       []
      lowercase							    =       ""
      lowercase							    =       False
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
          if not prev_is_special:
            out_string += " "

          out_string += self.sp_model.decode(__A       ) + token
          lowercase							    =       True
          lowercase							    =       []
        else:
          current_sub_tokens.append(__A       )
          lowercase							    =       False
      out_string += self.sp_model.decode(__A       )

      return out_string







    def 			UpperCamelCase_      (			self       ):
      lowercase							    =       {self.convert_ids_to_tokens(__A       ): i for i in range(self.vocab_size       )}
      vocab.update(self.added_tokens_encoder       )
      return vocab







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase = None       ):





      if not os.path.isdir(__A       ):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'       )
        return
      lowercase							    =       os.path.join(
          __A							,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

      if os.path.abspath(self.vocab_file       ) != os.path.abspath(__A       ) and os.path.isfile(self.vocab_file       ):
        copyfile(self.vocab_file							,		__A       )
      elif not os.path.isfile(self.vocab_file       ):
        with open(__A							,		'wb'       ) as fi:
          lowercase							    =       self.sp_model.serialized_model_proto()
          fi.write(__A       )

      return (out_vocab_file,)







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase							,		_lowerCamelCase = False       ):





      if isinstance(__A							,		__A       ):
        lowercase							    =       self.preprocess_text(__A       )
        lowercase							    =       self.sp_model.encode(__A       )
      else:
        lowercase							    =       [self.preprocess_text(__A       ) for t in text]
        lowercase							    =       self.sp_model.encode(__A       )

      if return_tensors is True or return_tensors == "pt":
        lowercase							    =       torch.tensor(__A       )

      return token_ids







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      return self.sp_model.decode(__A       )







    def 			UpperCamelCase_      (			self							,		_lowerCamelCase       ):
      lowercase							    =       [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
      lowercase							    =       (
          F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(__A       ) + F'{self.bos_token}Bot:'
      )
      return self.encode(text=__A       )






import argparse


_snake_case             =     '''docs/source/_static/js/custom.js'''






def 		lowercase_(  SCREAMING_SNAKE_CASE_	):


  '''simple docstring'''

  with open(SCREAMING_SNAKE_CASE_      ,						encoding="utf-8"      ,						newline="\n"	) as f:
    lowerCamelCase :					List[str]    =				f.readlines()
  lowerCamelCase :					int    =				0

  # First let's put the right version
  while not lines[index].startswith("const stableVersion ="	):
    index += 1
  lowerCamelCase :					str    =				f"""const stableVersion = \"v{version}\"\n"""

  # Then update the dictionary
  while not lines[index].startswith("const versionMapping = {"	):
    index += 1

  # We go until the end
  while not lines[index].startswith("}"	):
    index += 1
  # We add the new version at the end
  lines[index - 1] += f"""    \"v{version}\": \"v{version}\",\n"""

  with open(SCREAMING_SNAKE_CASE_      ,						"w"      ,						encoding="utf-8"      ,						newline="\n"	) as f:
    f.writelines(SCREAMING_SNAKE_CASE_	)


if __name__ == "__main__":
   _snake_case             =     argparse.ArgumentParser()
   parser.add_argument('''--version''', help='''Release version.''')
   _snake_case             =     parser.parse_args()
   update_custom_js(args.version)



