






"""simple docstring"""




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_       :							List[str]             =				logging.get_logger(__name__)

UpperCAmelCase_       :							Any             =				'''▁'''

UpperCAmelCase_       :							str             =				{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

UpperCAmelCase_       :							Optional[int]             =				{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

UpperCAmelCase_       :							Dict             =				{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

UpperCAmelCase_       :							Optional[int]             =				{
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

UpperCAmelCase_       :							str             =				{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}


class 	__UpperCAmelCase		(				_a	):


	'''simple docstring'''





	lowercase		:	List[str]							     =				["input_ids"]

	lowercase		:	Dict							     =				VOCAB_FILES_NAMES
	lowercase		:	Optional[int]							     =				PRETRAINED_INIT_CONFIGURATION
	lowercase		:	Any							     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase		:	Optional[Any]							     =				PRETRAINED_VOCAB_FILES_MAP
	lowercase		:	List[str]							     =				RESOURCE_FILES_NAMES


	def __init__(     self      ,      _A      ,      _A=None      ,      _A=False      ,      _A="utf8"      ,      _A="[UNK]"      ,      _A="[SEP]"      ,      _A="[PAD]"      ,      _A="[CLS]"      ,      _A="[MASK]"      ,      _A = None      ,      **_A      ,      ):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						={} if sp_model_kwargs is None else sp_model_kwargs
		super().__init__(
		    do_lower_case=__lowerCAmelCase      ,      unk_token=__lowerCAmelCase      ,      sep_token=__lowerCAmelCase      ,      pad_token=__lowerCAmelCase      ,      cls_token=__lowerCAmelCase      ,      mask_token=__lowerCAmelCase      ,      vocab_file=__lowerCAmelCase      ,      encoding=__lowerCAmelCase      ,      sp_model_kwargs=self.sp_model_kwargs      ,      **__lowerCAmelCase      ,      )
		_SCREAMING_SNAKE_CASE						=do_lower_case
		_SCREAMING_SNAKE_CASE						=sentencepiece_model_ckpt
		_SCREAMING_SNAKE_CASE						=spm.SentencePieceProcessor(**self.sp_model_kwargs					)
		self.sp_model.Load(__lowerCAmelCase					)

		# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
		if vocab_file is not None:
			_SCREAMING_SNAKE_CASE						=self.load_vocab(filepath=__lowerCAmelCase					)
		else:
			_SCREAMING_SNAKE_CASE						={self.sp_model.id_to_piece(__lowerCAmelCase					): id for id in range(self.sp_model.get_piece_size()					)}
		_SCREAMING_SNAKE_CASE						={v: k for k, v in self.vocab.items()}


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if text is None:
			return None

		_SCREAMING_SNAKE_CASE						=self.tokenize(__lowerCAmelCase					)
		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						='''''', []

		for i, ch in enumerate(__lowerCAmelCase					):
			if ch in self.SP_CHAR_MAPPING:
				_SCREAMING_SNAKE_CASE						=self.SP_CHAR_MAPPING.get(__lowerCAmelCase					)
			else:
				_SCREAMING_SNAKE_CASE						=unicodedata.normalize('''NFKC'''      ,      __lowerCAmelCase					)
			if self.is_whitespace(__lowerCAmelCase					):
				continue
			normalized_text += ch
			char_mapping.extend([i] * len(__lowerCAmelCase					)					)

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=normalized_text, [], 0

		if self.do_lower_case:
			_SCREAMING_SNAKE_CASE						=text.lower()

		for token in split_tokens:
			if token[:1] == "▁":
				_SCREAMING_SNAKE_CASE						=token[1:]
			_SCREAMING_SNAKE_CASE						=text[offset:].index(__lowerCAmelCase					) + offset
			_SCREAMING_SNAKE_CASE						=start + len(__lowerCAmelCase					)

			token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
			_SCREAMING_SNAKE_CASE						=end
		return token_mapping


	@property
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		return len(self.vocab					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		return dict(self.vocab      ,      **self.added_tokens_encoder					)


	def __getstate__(     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.__dict__.copy()
		_SCREAMING_SNAKE_CASE						=None
		return state


	def __setstate__(     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=d

		# for backward compatibility
		if not hasattr(self      ,      '''sp_model_kwargs'''					):
			_SCREAMING_SNAKE_CASE						={}

		_SCREAMING_SNAKE_CASE						=spm.SentencePieceProcessor(**self.sp_model_kwargs					)
		self.sp_model.Load(self.sentencepiece_model_ckpt					)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		return "".join((self.SP_CHAR_MAPPING.get(__lowerCAmelCase      ,      __lowerCAmelCase					) for c in text)					)


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A=False      ,      _A=6_4      ,      _A=0.1					):



		'''simple docstring'''

		if self.sp_model_kwargs.get('''enable_sampling'''					) is True:
			_SCREAMING_SNAKE_CASE						=True
		if self.sp_model_kwargs.get('''alpha'''					) is not None:
			_SCREAMING_SNAKE_CASE						=self.sp_model_kwargs.get('''alpha'''					)
		if self.sp_model_kwargs.get('''nbest_size'''					) is not None:
			_SCREAMING_SNAKE_CASE						=self.sp_model_kwargs.get('''nbest_size'''					)

		if not enable_sampling:
			_SCREAMING_SNAKE_CASE						=self.sp_model.EncodeAsPieces(__lowerCAmelCase					)
		else:
			_SCREAMING_SNAKE_CASE						=self.sp_model.SampleEncodeAsPieces(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase					)
		_SCREAMING_SNAKE_CASE						=[]
		for pi, piece in enumerate(__lowerCAmelCase					):
			if piece == SPIECE_UNDERLINE:
				if not pieces[pi + 1].startswith(__lowerCAmelCase					) and pi != 0:
					new_pieces.append(__lowerCAmelCase					)
					continue
				else:
					continue
			_SCREAMING_SNAKE_CASE						=0
			for i, chunk in enumerate(__lowerCAmelCase					):
				if chunk == SPIECE_UNDERLINE:
					continue
				if self.is_ch_char(__lowerCAmelCase					) or self.is_punct(__lowerCAmelCase					):
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]					)
					new_pieces.append(__lowerCAmelCase					)
					_SCREAMING_SNAKE_CASE						=i + 1
				elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]					)
					_SCREAMING_SNAKE_CASE						=i
				elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
					if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
						new_pieces.append(piece[lst_i:i]					)
					_SCREAMING_SNAKE_CASE						=i
			if len(__lowerCAmelCase					) > lst_i:
				new_pieces.append(piece[lst_i:]					)
		return new_pieces


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=''''''.join(__lowerCAmelCase					).replace(__lowerCAmelCase      ,      ''' '''					).strip()
		return out_string


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.convert_ids_to_tokens(__lowerCAmelCase					)
		_SCREAMING_SNAKE_CASE						=''''''.join(__lowerCAmelCase					).replace(__lowerCAmelCase      ,      ''' '''					).strip()
		return out_string


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		return self.vocab.get(__lowerCAmelCase      ,      self.vocab.get(self.unk_token					)					)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		return self.reverse_vocab.get(__lowerCAmelCase      ,      self.unk_token					)


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A=None					):



		'''simple docstring'''

		if token_ids_a is None:
			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
		_SCREAMING_SNAKE_CASE						=[self.cls_token_id]
		_SCREAMING_SNAKE_CASE						=[self.sep_token_id]
		return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A=None					):



		'''simple docstring'''

		if offset_mapping_a is None:
			return [(0, 0)] + offset_mapping_a + [(0, 0)]

		return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A=None      ,      _A=False					):



		'''simple docstring'''

		if already_has_special_tokens:
			if token_ids_a is not None:
				raise ValueError(
				    '''You should not supply a second sequence if the provided sequence of '''
				    '''ids is already formatted with special tokens for the model.'''					)
			return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

		if token_ids_a is not None:
			return [1] + ([0] * len(__lowerCAmelCase					)) + [1, 1] + ([0] * len(__lowerCAmelCase					)) + [1]
		return [1] + ([0] * len(__lowerCAmelCase					)) + [1]


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A = None					):



		'''simple docstring'''

		if token_ids_a is None:
			# [CLS] X [SEP]
			return (len(__lowerCAmelCase					) + 2) * [0]

		# [CLS] A [SEP] [SEP] B [SEP]
		return [0] * (len(__lowerCAmelCase					) + 1) + [1] * (len(__lowerCAmelCase					) + 3)


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if "\u4e00" <= char <= "\u9fff":
			return True
		return False


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if ("a" <= char <= "z") or ("A" <= char <= "Z"):
			return True
		return False


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if char in ",;:.?!~，；：。？！《》【】":
			return True
		return False


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		if char == " " or char == "\t" or char == "\n" or char == "\r":
			return True
		if len(__lowerCAmelCase					) == 1:
			_SCREAMING_SNAKE_CASE						=unicodedata.category(__lowerCAmelCase					)
			if cat == "Zs":
				return True
		return False


	def 				UpperCamelCase_  (     self      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						={}
		with io.open(__lowerCAmelCase      ,      '''r'''      ,      encoding='''utf-8'''					) as f:
			for index, line in enumerate(__lowerCAmelCase					):
				_SCREAMING_SNAKE_CASE						=line.rstrip('''\n'''					)
				_SCREAMING_SNAKE_CASE						=int(__lowerCAmelCase					)

		return token_to_idx







	def 				UpperCamelCase_  (     self      ,      _A      ,      _A = None					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=0
		if os.path.isdir(__lowerCAmelCase					):
			_SCREAMING_SNAKE_CASE						=os.path.join(
			    __lowerCAmelCase      ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
		else:
			_SCREAMING_SNAKE_CASE						=(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
		with open(__lowerCAmelCase      ,      '''w'''      ,      encoding='''utf-8'''					) as writer:
			for token, token_index in sorted(self.vocab.items()      ,      key=lambda _A					: kv[1]					):
				if index != token_index:
					logger.warning(
					    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
					    ''' Please check that the vocabulary is not corrupted!'''					)
					_SCREAMING_SNAKE_CASE						=token_index
				writer.write(token + '''\n'''					)
				index += 1

		_SCREAMING_SNAKE_CASE						=os.path.join(__lowerCAmelCase      ,      '''sentencepiece.bpe.model'''					)
		with open(__lowerCAmelCase      ,      '''wb'''					) as fi:
			_SCREAMING_SNAKE_CASE						=self.sp_model.serialized_model_proto()
			fi.write(__lowerCAmelCase					)

		return (vocab_file,)











import unittest

import torch

from diffusers import VQModel
from diffusers.utils import floats_tensor, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()

class 							__SCREAMING_SNAKE_CASE       (  _a ,		_a ,		unittest.TestCase							):
							snake_case  :   List[str]      =							VQModel
							snake_case  :   Dict      =							"""sample"""






							@property
							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase=(32, 32)	):
														UpperCamelCase__					  =      4
														UpperCamelCase__					  =      3

														UpperCamelCase__					  =      floats_tensor((batch_size, num_channels) + sizes	).to(__lowerCAmelCase	)

														return {"sample": image}






							@property
							def 					_lowerCamelCase		(    self	):
														return (3, 32, 32)






							@property
							def 					_lowerCamelCase		(    self	):
														return (3, 32, 32)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      {
														    """block_out_channels""": [32, 64],
														    """in_channels""": 3,
														    """out_channels""": 3,
														    """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
														    """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
														    """latent_channels""": 3,
														}
														UpperCamelCase__					  =      self.dummy_input
														return init_dict, inputs_dict






							def 					_lowerCamelCase		(    self	):
														pass






							def 					_lowerCamelCase		(    self	):
														pass






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__  ,  UpperCamelCase__					  =      VQModel.from_pretrained("""fusing/vqgan-dummy"""	,			output_loading_info=__lowerCAmelCase	)
														self.assertIsNotNone(__lowerCAmelCase	)
														self.assertEqual(len(loading_info["""missing_keys"""]	)	,			0	)

														model.to(__lowerCAmelCase	)
														UpperCamelCase__					  =      model(**self.dummy_input	)

														assert image is not None, "Make sure output is not None"







							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      VQModel.from_pretrained("""fusing/vqgan-dummy"""	)
														model.to(__lowerCAmelCase	).eval()

														torch.manual_seed(0	)
														if torch.cuda.is_available():
																					torch.cuda.manual_seed_all(0	)

														UpperCamelCase__					  =      torch.randn(1	,			model.config.in_channels	,			model.config.sample_size	,			model.config.sample_size	)
														UpperCamelCase__					  =      image.to(__lowerCAmelCase	)
														with torch.no_grad():
																					UpperCamelCase__					  =      model(__lowerCAmelCase	).sample

														UpperCamelCase__					  =      output[0, -1, -3:, -3:].flatten().cpu()
														# fmt: off
														UpperCamelCase__					  =      torch.tensor([-0.0153, -0.4044, -0.1880, -0.5161, -0.2418, -0.4072, -0.1612, -0.0633, -0.0143]	)
														# fmt: on
														self.assertTrue(torch.allclose(__lowerCAmelCase	,			__lowerCAmelCase	,			atol=1E-3	)	)




