
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_							=     logging.get_logger(__name__)
lowerCamelCase_							=     {"vocab_file": "spiece.model"}

lowerCamelCase_							=     {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

lowerCamelCase_							=     {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

lowerCamelCase_							=     "▁"




class 	__A(				__lowerCamelCase ):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__			    =			VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__			    =			PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__			    =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







  def __init__(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=True       ,			SCREAMING_SNAKE_CASE_=False       ,			SCREAMING_SNAKE_CASE_="[CLS]"       ,			SCREAMING_SNAKE_CASE_="[SEP]"       ,			SCREAMING_SNAKE_CASE_="<unk>"       ,			SCREAMING_SNAKE_CASE_="[SEP]"       ,			SCREAMING_SNAKE_CASE_="<pad>"       ,			SCREAMING_SNAKE_CASE_="[CLS]"       ,			SCREAMING_SNAKE_CASE_="[MASK]"       ,			SCREAMING_SNAKE_CASE_ = None       ,			**SCREAMING_SNAKE_CASE_       ,			):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.
       UpperCamelCase__				=				(
           AddedToken(SCREAMING_SNAKE_CASE_       ,			lstrip=SCREAMING_SNAKE_CASE_       ,			rstrip=SCREAMING_SNAKE_CASE_       ,			normalized=SCREAMING_SNAKE_CASE_						)
           if isinstance(SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_						)
           else mask_token
       )

       UpperCamelCase__				=				{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=SCREAMING_SNAKE_CASE_       ,			remove_space=SCREAMING_SNAKE_CASE_       ,			keep_accents=SCREAMING_SNAKE_CASE_       ,			bos_token=SCREAMING_SNAKE_CASE_       ,			eos_token=SCREAMING_SNAKE_CASE_       ,			unk_token=SCREAMING_SNAKE_CASE_       ,			sep_token=SCREAMING_SNAKE_CASE_       ,			pad_token=SCREAMING_SNAKE_CASE_       ,			cls_token=SCREAMING_SNAKE_CASE_       ,			mask_token=SCREAMING_SNAKE_CASE_       ,			sp_model_kwargs=self.sp_model_kwargs       ,			**SCREAMING_SNAKE_CASE_       ,			)

       UpperCamelCase__				=				do_lower_case
       UpperCamelCase__				=				remove_space
       UpperCamelCase__				=				keep_accents
       UpperCamelCase__				=				vocab_file

       UpperCamelCase__				=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(SCREAMING_SNAKE_CASE_						)







  @property
  def        UpperCAmelCase_					(self						):
       return len(self.sp_model						)







  def        UpperCAmelCase_					(self						):
       UpperCamelCase__				=				{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab







  def __getstate__(self						):
       UpperCamelCase__				=				self.__dict__.copy()
       UpperCamelCase__				=				None
       return state







  def __setstate__(self       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				d

       # for backward compatibility
       if not hasattr(self       ,			"""sp_model_kwargs"""						):
            UpperCamelCase__				=				{}

       UpperCamelCase__				=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(self.vocab_file						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_						):
       if self.remove_space:
            UpperCamelCase__				=				""" """.join(inputs.strip().split()						)
       else:
            UpperCamelCase__				=				inputs
       UpperCamelCase__				=				outputs.replace("""``"""       ,			"""\""""						).replace("""''"""       ,			"""\""""						)

       if not self.keep_accents:
            UpperCamelCase__				=				unicodedata.normalize("""NFKD"""       ,			SCREAMING_SNAKE_CASE_						)
            UpperCamelCase__				=				"""""".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE_						)]						)
       if self.do_lower_case:
            UpperCamelCase__				=				outputs.lower()

       return outputs







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				self.preprocess_text(SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				self.sp_model.encode(SCREAMING_SNAKE_CASE_       ,			out_type=SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=				[]
       for piece in pieces:
            if len(SCREAMING_SNAKE_CASE_						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
                 UpperCamelCase__				=				self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE_       ,			""""""						)						)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]						) == 1:
                           UpperCamelCase__				=				cur_pieces[1:]
                      else:
                           UpperCamelCase__				=				cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]						)
                 new_pieces.extend(SCREAMING_SNAKE_CASE_						)
            else:
                 new_pieces.append(SCREAMING_SNAKE_CASE_						)

       return new_pieces







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_						):
       return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_						):
       return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE_						)







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=				[]
       UpperCamelCase__				=				""""""
       UpperCamelCase__				=				False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						) + token
                 UpperCamelCase__				=				True
                 UpperCamelCase__				=				[]
            else:
                 current_sub_tokens.append(SCREAMING_SNAKE_CASE_						)
                 UpperCamelCase__				=				False
       out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						)
       return out_string.strip()







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=				[self.sep_token_id]
       UpperCamelCase__				=				[self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_ = None       ,			SCREAMING_SNAKE_CASE_ = False						):

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE_       ,			token_ids_a=SCREAMING_SNAKE_CASE_       ,			already_has_special_tokens=SCREAMING_SNAKE_CASE_						)

       if token_ids_a is not None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=				[self.sep_token_id]
       UpperCamelCase__				=				[self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







  def        UpperCAmelCase_					(self       ,			SCREAMING_SNAKE_CASE_       ,			SCREAMING_SNAKE_CASE_ = None						):
       if not os.path.isdir(SCREAMING_SNAKE_CASE_						):
            logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
            return
       UpperCamelCase__				=				os.path.join(
           SCREAMING_SNAKE_CASE_       ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE_						) and os.path.isfile(self.vocab_file						):
            copyfile(self.vocab_file       ,			SCREAMING_SNAKE_CASE_						)
       elif not os.path.isfile(self.vocab_file						):
            with open(SCREAMING_SNAKE_CASE_       ,			"""wb"""						) as fi:
                 UpperCamelCase__				=				self.sp_model.serialized_model_proto()
                 fi.write(SCREAMING_SNAKE_CASE_						)

       return (out_vocab_file,)












import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     GPTSanJapaneseTokenizer
       lowerCAmelCase__      		=     False
       lowerCAmelCase__      		=     {"""do_clean_text""": False, """add_prefix_space""": False}




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         super().setUp()

         # fmt: off
         __snake_case:						str   =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         __snake_case:						List[Any]   =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         __snake_case:						Optional[int]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						Any   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						int   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""emoji_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
         with open(self.emoji_file							,				"""w"""		) as emoji_writer:
           emoji_writer.write(json.dumps(A		)		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Dict		):
         __snake_case:						Tuple   =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
         __snake_case:						str   =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Optional[int]		):
         __snake_case  ,   __snake_case:						Optional[int]   =						self.get_input_output_texts(A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_special_tokens=A		)
         __snake_case:						int   =						tokenizer.decode(A							,				clean_up_tokenization_spaces=A		)
         return text, ids




       def 							UpperCAmelCase__ (  self :    int		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Dict		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Dict   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						List[str]   =						"""こんにちは、世界。　こんばんは、㔺界。"""
         __snake_case:						Optional[int]   =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         __snake_case:						Any   =						tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						str   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						Optional[int]   =						tokens + [tokenizer.unk_token]
         __snake_case:						List[str]   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    str		):
         __snake_case:						Union[str, Any]   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						Optional[Any]   =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         __snake_case:						Optional[Any]   =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
         __snake_case:						Union[str, Any]   =						tokenizer.encode(A		)
         __snake_case:						List[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Any		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Any   =						"""こんにちは、世界。"""
         __snake_case:						Tuple   =						"""こんばんは、㔺界。😀"""
         __snake_case:						Optional[Any]   =						"""こんにちは、世界。こんばんは、世界。😀"""
         __snake_case:						int   =						tokenizer.encode(prefix_text + input_text		)
         __snake_case:						Union[str, Any]   =						tokenizer.encode(""""""							,				prefix_text=prefix_text + input_text		)
         __snake_case:						Tuple   =						tokenizer.encode(A							,				prefix_text=A		)
         __snake_case:						Union[str, Any]   =						tokenizer.decode(A		)
         __snake_case:						Dict   =						tokenizer.decode(A		)
         __snake_case:						Optional[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						int   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Optional[int]   =						"""こんにちは、世界。"""
         __snake_case:						Any   =						"""こんばんは、㔺界。😀"""

         __snake_case:						Optional[int]   =						len(tokenizer.encode(A		)		) - 2
         __snake_case:						str   =						len(tokenizer.encode(A		)		) - 2

         __snake_case:						Dict   =						[1] + [0] * (len_prefix + len_text + 1)
         __snake_case:						str   =						[1] * (len_prefix + len_text + 1) + [0]
         __snake_case:						List[Any]   =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __snake_case:						int   =						tokenizer(prefix_text + input_text		).token_type_ids
         __snake_case:						Optional[int]   =						tokenizer(""""""							,				prefix_text=prefix_text + input_text		).token_type_ids
         __snake_case:						Tuple   =						tokenizer(A							,				prefix_text=A		).token_type_ids
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Dict		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						int   =						tokenizer.encode("""あンいワ"""		)
         __snake_case:						Optional[int]   =						tokenizer.encode(""""""							,				prefix_text="""あンいワ"""		)
         __snake_case:						List[Any]   =						tokenizer.encode("""いワ"""							,				prefix_text="""あン"""		)

         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertNotEqual(A							,				A		)
         self.assertNotEqual(A							,				A		)
         self.assertEqual(x_token_a[1]							,				x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]							,				x_token_a[3]		)  # SEG token




       @slow
       def 							UpperCAmelCase__ (  self :    int		):
         __snake_case:						List[str]   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						Union[str, Any]   =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         __snake_case:						Optional[int]   =						tokenizer(A							,				padding=A		)
         __snake_case:						int   =						tokenizer.batch_encode_plus(A							,				padding=A		)

         # fmt: off
         __snake_case:						List[str]   =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
         __snake_case:						int   =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __snake_case:						Dict   =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids							,				A		)
         self.assertListEqual(x_token.token_type_ids							,				A		)
         self.assertListEqual(x_token.attention_mask							,				A		)
         self.assertListEqual(x_token_a.input_ids							,				A		)
         self.assertListEqual(x_token_a.token_type_ids							,				A		)
         self.assertListEqual(x_token_a.attention_mask							,				A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






       def 							UpperCAmelCase__ (  self :    List[str]		):
         # tokenizer has no padding token
         pass






