





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)
UpperCamelCase__           =			{"vocab_file": "spiece.model"}

UpperCamelCase__           =			{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

UpperCamelCase__           =			{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

UpperCamelCase__           =			"▁"

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Union[str, Any]      =							VOCAB_FILES_NAMES
							snake_case  :   Union[str, Any]      =							PRETRAINED_VOCAB_FILES_MAP
							snake_case  :   int      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase=True	,			__lowerCAmelCase=True	,			__lowerCAmelCase=False	,			__lowerCAmelCase="[CLS]"	,			__lowerCAmelCase="[SEP]"	,			__lowerCAmelCase="<unk>"	,			__lowerCAmelCase="[SEP]"	,			__lowerCAmelCase="<pad>"	,			__lowerCAmelCase="[CLS]"	,			__lowerCAmelCase="[MASK]"	,			__lowerCAmelCase = None	,			**__lowerCAmelCase	,			):
														# Mask token behave like a normal word, i.e. include the space before it and
														# is included in the raw text, there should be a match in a non-normalized sentence.
														UpperCamelCase__					  =      (
														    AddedToken(__lowerCAmelCase	,			lstrip=__lowerCAmelCase	,			rstrip=__lowerCAmelCase	,			normalized=__lowerCAmelCase	)
														    if isinstance(__lowerCAmelCase	,			__lowerCAmelCase	)
														    else mask_token
														)

														UpperCamelCase__					  =      {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=__lowerCAmelCase	,			remove_space=__lowerCAmelCase	,			keep_accents=__lowerCAmelCase	,			bos_token=__lowerCAmelCase	,			eos_token=__lowerCAmelCase	,			unk_token=__lowerCAmelCase	,			sep_token=__lowerCAmelCase	,			pad_token=__lowerCAmelCase	,			cls_token=__lowerCAmelCase	,			mask_token=__lowerCAmelCase	,			sp_model_kwargs=self.sp_model_kwargs	,			**__lowerCAmelCase	,			)

														UpperCamelCase__					  =      do_lower_case
														UpperCamelCase__					  =      remove_space
														UpperCamelCase__					  =      keep_accents
														UpperCamelCase__					  =      vocab_file

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(__lowerCAmelCase	)






							@property
							def 					_lowerCamelCase		(    self	):
														return len(self.sp_model	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      {self.convert_ids_to_tokens(__lowerCAmelCase	): i for i in range(self.vocab_size	)}
														vocab.update(self.added_tokens_encoder	)
														return vocab






							def __getstate__(    self	):
														UpperCamelCase__					  =      self.__dict__.copy()
														UpperCamelCase__					  =      None
														return state






							def __setstate__(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      d

														# for backward compatibility
														if not hasattr(self	,			"""sp_model_kwargs"""	):
																					UpperCamelCase__					  =      {}

														UpperCamelCase__					  =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(self.vocab_file	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														if self.remove_space:
																					UpperCamelCase__					  =      """ """.join(inputs.strip().split()	)
														else:
																					UpperCamelCase__					  =      inputs
														UpperCamelCase__					  =      outputs.replace("""``"""	,			"""\""""	).replace("""''"""	,			"""\""""	)

														if not self.keep_accents:
																					UpperCamelCase__					  =      unicodedata.normalize("""NFKD"""	,			__lowerCAmelCase	)
																					UpperCamelCase__					  =      """""".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase	)]	)
														if self.do_lower_case:
																					UpperCamelCase__					  =      outputs.lower()

														return outputs






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      self.preprocess_text(__lowerCAmelCase	)
														UpperCamelCase__					  =      self.sp_model.encode(__lowerCAmelCase	,			out_type=__lowerCAmelCase	)
														UpperCamelCase__					  =      []
														for piece in pieces:
																					if len(__lowerCAmelCase	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
																												UpperCamelCase__					  =      self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase	,			""""""	)	)
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]	) == 1:
																																										UpperCamelCase__					  =      cur_pieces[1:]
																																			else:
																																										UpperCamelCase__					  =      cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]	)
																												new_pieces.extend(__lowerCAmelCase	)
																					else:
																												new_pieces.append(__lowerCAmelCase	)

														return new_pieces






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.PieceToId(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														return self.sp_model.IdToPiece(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      []
														UpperCamelCase__					  =      """"""
														UpperCamelCase__					  =      False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(__lowerCAmelCase	) + token
																												UpperCamelCase__					  =      True
																												UpperCamelCase__					  =      []
																					else:
																												current_sub_tokens.append(__lowerCAmelCase	)
																												UpperCamelCase__					  =      False
														out_string += self.sp_model.decode(__lowerCAmelCase	)
														return out_string.strip()






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	,			__lowerCAmelCase = False	):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase	,			token_ids_a=__lowerCAmelCase	,			already_has_special_tokens=__lowerCAmelCase	)

														if token_ids_a is not None:
																					return [1] + ([0] * len(__lowerCAmelCase	)) + [1] + ([0] * len(__lowerCAmelCase	)) + [1]
														return [1] + ([0] * len(__lowerCAmelCase	)) + [1]






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														UpperCamelCase__					  =      [self.sep_token_id]
														UpperCamelCase__					  =      [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep	) * [0]
														return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	,			__lowerCAmelCase = None	):
														if not os.path.isdir(__lowerCAmelCase	):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
																					return
														UpperCamelCase__					  =      os.path.join(
														    __lowerCAmelCase	,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

														if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCAmelCase	) and os.path.isfile(self.vocab_file	):
																					copyfile(self.vocab_file	,			__lowerCAmelCase	)
														elif not os.path.isfile(self.vocab_file	):
																					with open(__lowerCAmelCase	,			"""wb"""	) as fi:
																												UpperCamelCase__					  =      self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase	)

														return (out_vocab_file,)











import unittest

import torch

from diffusers import VQModel
from diffusers.utils import floats_tensor, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()

class 							__SCREAMING_SNAKE_CASE       (  _a ,		_a ,		unittest.TestCase							):
							snake_case  :   List[str]      =							VQModel
							snake_case  :   Dict      =							"""sample"""






							@property
							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase=(32, 32)	):
														UpperCamelCase__					  =      4
														UpperCamelCase__					  =      3

														UpperCamelCase__					  =      floats_tensor((batch_size, num_channels) + sizes	).to(__lowerCAmelCase	)

														return {"sample": image}






							@property
							def 					_lowerCamelCase		(    self	):
														return (3, 32, 32)






							@property
							def 					_lowerCamelCase		(    self	):
														return (3, 32, 32)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      {
														    """block_out_channels""": [32, 64],
														    """in_channels""": 3,
														    """out_channels""": 3,
														    """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
														    """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
														    """latent_channels""": 3,
														}
														UpperCamelCase__					  =      self.dummy_input
														return init_dict, inputs_dict






							def 					_lowerCamelCase		(    self	):
														pass






							def 					_lowerCamelCase		(    self	):
														pass






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__  ,  UpperCamelCase__					  =      VQModel.from_pretrained("""fusing/vqgan-dummy"""	,			output_loading_info=__lowerCAmelCase	)
														self.assertIsNotNone(__lowerCAmelCase	)
														self.assertEqual(len(loading_info["""missing_keys"""]	)	,			0	)

														model.to(__lowerCAmelCase	)
														UpperCamelCase__					  =      model(**self.dummy_input	)

														assert image is not None, "Make sure output is not None"







							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      VQModel.from_pretrained("""fusing/vqgan-dummy"""	)
														model.to(__lowerCAmelCase	).eval()

														torch.manual_seed(0	)
														if torch.cuda.is_available():
																					torch.cuda.manual_seed_all(0	)

														UpperCamelCase__					  =      torch.randn(1	,			model.config.in_channels	,			model.config.sample_size	,			model.config.sample_size	)
														UpperCamelCase__					  =      image.to(__lowerCAmelCase	)
														with torch.no_grad():
																					UpperCamelCase__					  =      model(__lowerCAmelCase	).sample

														UpperCamelCase__					  =      output[0, -1, -3:, -3:].flatten().cpu()
														# fmt: off
														UpperCamelCase__					  =      torch.tensor([-0.0153, -0.4044, -0.1880, -0.5161, -0.2418, -0.4072, -0.1612, -0.0633, -0.0143]	)
														# fmt: on
														self.assertTrue(torch.allclose(__lowerCAmelCase	,			__lowerCAmelCase	,			atol=1E-3	)	)




