






"""simple docstring"""




import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionInstructPixaPixPipeline,
    UNetaDConditionModel,
)
from diffusers.image_processor import VaeImageProcessor
from diffusers.utils import floats_tensor, load_image, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()






class        UpperCamelCase			(	snake_case_		,     snake_case_		,     snake_case_		,     unittest.TestCase      ):
 UpperCamelCase					:       str			=					StableDiffusionInstructPixaPixPipeline
 UpperCamelCase					:       List[str]			=					TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'''height''', '''width''', '''cross_attention_kwargs'''}
 UpperCamelCase					:       List[Any]			=					TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
 UpperCamelCase					:       Any			=					IMAGE_TO_IMAGE_IMAGE_PARAMS
 UpperCamelCase					:       Optional[Any]			=					IMAGE_TO_IMAGE_IMAGE_PARAMS


 def _lowercase	(     self		:						Any							)					->       Tuple:
   torch.manual_seed(0							)
   _a  :       Optional[Any]             =	UNetaDConditionModel(
       block_out_channels=(32, 64)							,						layers_per_block=2							,						sample_size=32							,						in_channels=8							,						out_channels=4							,						down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")							,						up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")							,						cross_attention_dim=32							,						)
   _a  :       Tuple             =	PNDMScheduler(skip_prk_steps=UpperCAmelCase__							)
   torch.manual_seed(0							)
   _a  :       List[str]             =	AutoencoderKL(
       block_out_channels=[32, 64]							,						in_channels=3							,						out_channels=3							,						down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]							,						up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]							,						latent_channels=4							,						)
   torch.manual_seed(0							)
   _a  :       Union[str, Any]             =	CLIPTextConfig(
       bos_token_id=0							,						eos_token_id=2							,						hidden_size=32							,						intermediate_size=37							,						layer_norm_eps=1E-05							,						num_attention_heads=4							,						num_hidden_layers=5							,						pad_token_id=1							,						vocab_size=1000							,						)
   _a  :       Union[str, Any]             =	CLIPTextModel(UpperCAmelCase__							)
   _a  :       List[Any]             =	CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""							)

   _a  :       Dict             =	{
       """unet""": unet,
       """scheduler""": scheduler,
       """vae""": vae,
       """text_encoder""": text_encoder,
       """tokenizer""": tokenizer,
       """safety_checker""": None,
       """feature_extractor""": None,
   }
   return components


 def _lowercase	(     self		:						Any							,						UpperCAmelCase__		:						str							,						UpperCAmelCase__		:						Any=0							)					->       Dict:
   _a  :       Tuple             =	floats_tensor((1, 3, 32, 32)							,						rng=random.Random(UpperCAmelCase__							)							).to(UpperCAmelCase__							)
   _a  :       List[Any]             =	image.cpu().permute(0							,						2							,						3							,						1							)[0]
   _a  :       int             =	Image.fromarray(np.uinta(UpperCAmelCase__							)							).convert("""RGB"""							)
   if str(UpperCAmelCase__							).startswith("""mps"""							):
     _a  :       Tuple             =	torch.manual_seed(UpperCAmelCase__							)
   else:
     _a  :       Tuple             =	torch.Generator(device=UpperCAmelCase__							).manual_seed(UpperCAmelCase__							)
   _a  :       List[str]             =	{
       """prompt""": """A painting of a squirrel eating a burger""",
       """image""": image,
       """generator""": generator,
       """num_inference_steps""": 2,
       """guidance_scale""": 6.0,
       """image_guidance_scale""": 1,
       """output_type""": """numpy""",
   }
   return inputs


 def _lowercase	(     self		:						Dict							)					->       Union[str, Any]:
   _a  :       List[str]             =	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
   _a  :       int             =	self.get_dummy_components()
   _a  :       List[str]             =	StableDiffusionInstructPixaPixPipeline(**UpperCAmelCase__							)
   _a  :       List[str]             =	sd_pipe.to(UpperCAmelCase__							)
   sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

   _a  :       Any             =	self.get_dummy_inputs(UpperCAmelCase__							)
   _a  :       List[Any]             =	sd_pipe(**UpperCAmelCase__							).images
   _a  :       Dict             =	image[0, -3:, -3:, -1]
   assert image.shape == (1, 32, 32, 3)
   _a  :       List[Any]             =	np.array([0.7_5_2_6, 0.3_7_5_0, 0.4_5_4_7, 0.6_1_1_7, 0.5_8_6_6, 0.5_0_1_6, 0.4_3_2_7, 0.5_6_4_2, 0.4_8_1_5]							)

   assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-3


 def _lowercase	(     self		:						int							)					->       Optional[Any]:
   _a  :       Tuple             =	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
   _a  :       str             =	self.get_dummy_components()
   _a  :       Optional[int]             =	StableDiffusionInstructPixaPixPipeline(**UpperCAmelCase__							)
   _a  :       str             =	sd_pipe.to(UpperCAmelCase__							)
   sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

   _a  :       Any             =	self.get_dummy_inputs(UpperCAmelCase__							)
   _a  :       Tuple             =	"""french fries"""
   _a  :       Dict             =	sd_pipe(**UpperCAmelCase__							,						negative_prompt=UpperCAmelCase__							)
   _a  :       Dict             =	output.images
   _a  :       str             =	image[0, -3:, -3:, -1]

   assert image.shape == (1, 32, 32, 3)
   _a  :       int             =	np.array([0.7_5_1_1, 0.3_6_4_2, 0.4_5_5_3, 0.6_2_3_6, 0.5_7_9_7, 0.5_0_1_3, 0.4_3_4_3, 0.5_6_1_1, 0.4_8_3_1]							)

   assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-3


 def _lowercase	(     self		:						Dict							)					->       List[Any]:
   _a  :       Union[str, Any]             =	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
   _a  :       Union[str, Any]             =	self.get_dummy_components()
   _a  :       Tuple             =	StableDiffusionInstructPixaPixPipeline(**UpperCAmelCase__							)
   _a  :       List[Any]             =	sd_pipe.to(UpperCAmelCase__							)
   sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

   _a  :       Any             =	self.get_dummy_inputs(UpperCAmelCase__							)
   _a  :       Union[str, Any]             =	[inputs["""prompt"""]] * 2

   _a  :       Union[str, Any]             =	np.array(inputs["""image"""]							).astype(np.floataa							) / 2_5_5.0
   _a  :       Optional[int]             =	torch.from_numpy(UpperCAmelCase__							).unsqueeze(0							).to(UpperCAmelCase__							)
   _a  :       Optional[int]             =	image / 2 + 0.5
   _a  :       Any             =	image.permute(0							,						3							,						1							,						2							)
   _a  :       str             =	image.repeat(2							,						1							,						1							,						1							)

   _a  :       Optional[int]             =	sd_pipe(**UpperCAmelCase__							).images
   _a  :       List[Any]             =	image[-1, -3:, -3:, -1]

   assert image.shape == (2, 32, 32, 3)
   _a  :       List[Any]             =	np.array([0.5_8_1_2, 0.5_7_4_8, 0.5_2_2_2, 0.5_9_0_8, 0.5_6_9_5, 0.7_1_7_4, 0.6_8_0_4, 0.5_5_2_3, 0.5_5_7_9]							)

   assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-3


 def _lowercase	(     self		:						Any							)					->       Union[str, Any]:
   _a  :       int             =	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
   _a  :       Union[str, Any]             =	self.get_dummy_components()
   _a  :       Union[str, Any]             =	EulerAncestralDiscreteScheduler(
       beta_start=0.0_0_0_8_5							,						beta_end=0.0_1_2							,						beta_schedule="""scaled_linear"""							)
   _a  :       Dict             =	StableDiffusionInstructPixaPixPipeline(**UpperCAmelCase__							)
   _a  :       str             =	sd_pipe.to(UpperCAmelCase__							)
   sd_pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

   _a  :       str             =	self.get_dummy_inputs(UpperCAmelCase__							)
   _a  :       List[str]             =	sd_pipe(**UpperCAmelCase__							).images
   _a  :       List[Any]             =	image[0, -3:, -3:, -1]

   _a  :       Dict             =	[round(UpperCAmelCase__							,						4							) for x in image_slice.flatten().tolist()]
   print(""",""".join([str(UpperCAmelCase__							) for x in slice]							)							)

   assert image.shape == (1, 32, 32, 3)
   _a  :       str             =	np.array([0.7_4_1_7, 0.3_8_4_2, 0.4_7_3_2, 0.5_7_7_6, 0.5_8_9_1, 0.5_1_3_9, 0.4_0_5_2, 0.5_6_7_3, 0.4_9_8_6]							)

   assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-3


 def _lowercase	(     self		:						Optional[int]							)					->       int:
   super().test_inference_batch_single_identical(expected_max_diff=3E-3							)







 def _lowercase	(     self		:						Tuple							)					->       Tuple:
   _a  :       int             =	self.get_dummy_components()
   _a  :       int             =	StableDiffusionInstructPixaPixPipeline(**UpperCAmelCase__							)
   _a  :       List[str]             =	VaeImageProcessor(do_resize=UpperCAmelCase__							,						do_normalize=UpperCAmelCase__							)
   _a  :       Union[str, Any]             =	pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

   _a  :       List[Any]             =	pipe(**self.get_dummy_inputs_by_type(UpperCAmelCase__							,						input_image_type="""pt"""							)							)[0]

   _a  :       str             =	components["""vae"""]
   _a  :       Optional[int]             =	self.get_dummy_inputs_by_type(UpperCAmelCase__							,						input_image_type="""pt"""							)

   for image_param in self.image_latents_params:
     if image_param in inputs.keys():
       _a  :       Any             =	vae.encode(inputs[image_param]							).latent_dist.mode()

   _a  :       List[str]             =	pipe(**UpperCAmelCase__							)[0]

   _a  :       Dict             =	np.abs(out - out_latents_inputs							).max()
   self.assertLess(UpperCAmelCase__							,						1E-4							,						"""passing latents as image input generate different result from passing image"""							)







@slow
@require_torch_gpu
class        UpperCamelCase			(	unittest.TestCase      ):


 def _lowercase	(     self		:						int							)					->       Optional[int]:
   super().tearDown()
   gc.collect()
   torch.cuda.empty_cache()


 def _lowercase	(     self		:						int							,						UpperCAmelCase__		:						List[Any]=0							)					->       str:
   _a  :       str             =	torch.manual_seed(UpperCAmelCase__							)
   _a  :       Any             =	load_image(
       """https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_pix2pix/example.jpg"""							)
   _a  :       Union[str, Any]             =	{
       """prompt""": """turn him into a cyborg""",
       """image""": image,
       """generator""": generator,
       """num_inference_steps""": 3,
       """guidance_scale""": 7.5,
       """image_guidance_scale""": 1.0,
       """output_type""": """numpy""",
   }
   return inputs


 def _lowercase	(     self		:						str							)					->       str:
   _a  :       Optional[int]             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       """timbrooks/instruct-pix2pix"""							,						safety_checker=UpperCAmelCase__							)
   pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing()

   _a  :       Dict             =	self.get_inputs()
   _a  :       Any             =	pipe(**UpperCAmelCase__							).images
   _a  :       Tuple             =	image[0, -3:, -3:, -1].flatten()

   assert image.shape == (1, 512, 512, 3)
   _a  :       Tuple             =	np.array([0.5_9_0_2, 0.6_0_1_5, 0.6_0_2_7, 0.5_9_8_3, 0.6_0_9_2, 0.6_0_6_1, 0.5_7_6_5, 0.5_7_8_5, 0.5_5_5_5]							)

   assert np.abs(expected_slice - image_slice							).max() < 1E-3


 def _lowercase	(     self		:						Dict							)					->       Any:
   _a  :       Dict             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       """timbrooks/instruct-pix2pix"""							,						safety_checker=UpperCAmelCase__							)
   _a  :       Union[str, Any]             =	LMSDiscreteScheduler.from_config(pipe.scheduler.config							)
   pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing()

   _a  :       List[Any]             =	self.get_inputs()
   _a  :       List[str]             =	pipe(**UpperCAmelCase__							).images
   _a  :       str             =	image[0, -3:, -3:, -1].flatten()

   assert image.shape == (1, 512, 512, 3)
   _a  :       List[Any]             =	np.array([0.6_5_7_8, 0.6_8_1_7, 0.6_9_7_2, 0.6_7_6_1, 0.6_8_5_6, 0.6_9_1_6, 0.6_4_2_8, 0.6_5_1_6, 0.6_3_0_1]							)

   assert np.abs(expected_slice - image_slice							).max() < 1E-3


 def _lowercase	(     self		:						Dict							)					->       Dict:
   _a  :       List[Any]             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       """timbrooks/instruct-pix2pix"""							,						safety_checker=UpperCAmelCase__							)
   _a  :       Union[str, Any]             =	DDIMScheduler.from_config(pipe.scheduler.config							)
   pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing()

   _a  :       Optional[Any]             =	self.get_inputs()
   _a  :       Optional[int]             =	pipe(**UpperCAmelCase__							).images
   _a  :       List[Any]             =	image[0, -3:, -3:, -1].flatten()

   assert image.shape == (1, 512, 512, 3)
   _a  :       Dict             =	np.array([0.3_8_2_8, 0.3_8_3_4, 0.3_8_1_8, 0.3_7_9_2, 0.3_8_6_5, 0.3_7_5_2, 0.3_7_9_2, 0.3_8_4_7, 0.3_7_5_3]							)

   assert np.abs(expected_slice - image_slice							).max() < 1E-3


 def _lowercase	(     self		:						Optional[int]							)					->       Optional[int]:
   _a  :       str             =	0

   def callback_fn(UpperCAmelCase__		:						int							,						UpperCAmelCase__		:						int							,						UpperCAmelCase__		:						torch.FloatTensor							) -> None:
     _a  :       str             =	True
     nonlocal number_of_steps
     number_of_steps += 1
     if step == 1:
       _a  :       Any             =	latents.detach().cpu().numpy()
       assert latents.shape == (1, 4, 64, 64)
       _a  :       List[Any]             =	latents[0, -3:, -3:, -1]
       _a  :       Dict             =	np.array([-0.2_4_6_3, -0.4_6_4_4, -0.9_7_5_6, 1.5_1_7_6, 1.4_4_1_4, 0.7_8_6_6, 0.9_8_9_7, 0.8_5_2_1, 0.7_9_8_3]							)

       assert np.abs(latents_slice.flatten() - expected_slice							).max() < 5E-2
     elif step == 2:
       _a  :       Union[str, Any]             =	latents.detach().cpu().numpy()
       assert latents.shape == (1, 4, 64, 64)
       _a  :       int             =	latents[0, -3:, -3:, -1]
       _a  :       Union[str, Any]             =	np.array([-0.2_6_4_4, -0.4_6_2_6, -0.9_6_5_3, 1.5_1_7_6, 1.4_5_5_1, 0.7_6_8_6, 0.9_8_0_5, 0.8_4_5_2, 0.8_1_1_5]							)

       assert np.abs(latents_slice.flatten() - expected_slice							).max() < 5E-2

   _a  :       Tuple             =	False

   _a  :       int             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       """timbrooks/instruct-pix2pix"""							,						safety_checker=UpperCAmelCase__							,						torch_dtype=torch.floataa							)
   _a  :       Dict             =	pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing()

   _a  :       Tuple             =	self.get_inputs()
   pipe(**UpperCAmelCase__							,						callback=UpperCAmelCase__							,						callback_steps=1							)
   assert callback_fn.has_been_called
   assert number_of_steps == 3


 def _lowercase	(     self		:						int							)					->       Optional[int]:
   torch.cuda.empty_cache()
   torch.cuda.reset_max_memory_allocated()
   torch.cuda.reset_peak_memory_stats()

   _a  :       List[str]             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       """timbrooks/instruct-pix2pix"""							,						safety_checker=UpperCAmelCase__							,						torch_dtype=torch.floataa							)
   _a  :       List[str]             =	pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing(1							)
   pipe.enable_sequential_cpu_offload()

   _a  :       Optional[int]             =	self.get_inputs()
   _a  :       Optional[Any]             =	pipe(**UpperCAmelCase__							)

   _a  :       Any             =	torch.cuda.max_memory_allocated()
   # make sure that less than 2.2 GB is allocated
   assert mem_bytes < 2.2 * 10**9







 def _lowercase	(     self		:						int							)					->       int:
   _a  :       Optional[Any]             =	self.get_inputs()
   # resize to resolution that is divisible by 8 but not 16 or 32
   _a  :       Dict             =	inputs["""image"""].resize((504, 504)							)

   _a  :       Tuple             =	"""timbrooks/instruct-pix2pix"""
   _a  :       Optional[int]             =	StableDiffusionInstructPixaPixPipeline.from_pretrained(
       UpperCAmelCase__							,						safety_checker=UpperCAmelCase__							,						)
   pipe.to(UpperCAmelCase__							)
   pipe.set_progress_bar_config(disable=UpperCAmelCase__							)
   pipe.enable_attention_slicing()

   _a  :       Dict             =	pipe(**UpperCAmelCase__							)
   _a  :       Tuple             =	output.images[0]

   _a  :       List[str]             =	image[255:258, 383:386, -1]

   assert image.shape == (504, 504, 3)
   _a  :       Union[str, Any]             =	np.array([0.2_7_2_6, 0.2_5_2_9, 0.2_6_6_4, 0.2_6_5_5, 0.2_6_4_1, 0.2_6_4_2, 0.2_5_9_1, 0.2_6_4_9, 0.2_5_9_0]							)

   assert np.abs(image_slice.flatten() - expected_slice							).max() < 5E-3









"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {'vocab_file': 'spiece.model'}

_snake_case          =   {
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}






class        UpperCamelCase			(	snake_case_      ):


 def __init__(     self		:						Tuple							,						UpperCAmelCase__		:						Optional[Any]							,						UpperCAmelCase__		:						int=False							,						UpperCAmelCase__		:						Dict=True							,						UpperCAmelCase__		:						Union[str, Any]=False							,						UpperCAmelCase__		:						Dict="<s>"							,						UpperCAmelCase__		:						Any="</s>"							,						UpperCAmelCase__		:						Any="<unk>"							,						UpperCAmelCase__		:						int="<sep>"							,						UpperCAmelCase__		:						Tuple="<pad>"							,						UpperCAmelCase__		:						Any="<cls>"							,						UpperCAmelCase__		:						Optional[Any]="<mask>"							,						UpperCAmelCase__		:						int=["<eop>", "<eod>"]							,						UpperCAmelCase__		:						Optional[Dict[str, Any]] = None							,						**UpperCAmelCase__		:						List[str]							,						)					->       None:
   _a  :       Optional[int]             =	AddedToken(UpperCAmelCase__							,						lstrip=UpperCAmelCase__							,						rstrip=UpperCAmelCase__							) if isinstance(UpperCAmelCase__							,						UpperCAmelCase__							) else mask_token

   _a  :       Union[str, Any]             =	{} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=UpperCAmelCase__							,						remove_space=UpperCAmelCase__							,						keep_accents=UpperCAmelCase__							,						bos_token=UpperCAmelCase__							,						eos_token=UpperCAmelCase__							,						unk_token=UpperCAmelCase__							,						sep_token=UpperCAmelCase__							,						pad_token=UpperCAmelCase__							,						cls_token=UpperCAmelCase__							,						mask_token=UpperCAmelCase__							,						additional_special_tokens=UpperCAmelCase__							,						sp_model_kwargs=self.sp_model_kwargs							,						**UpperCAmelCase__							,						)

   _a  :       Optional[Any]             =	3

   _a  :       Tuple             =	do_lower_case
   _a  :       Tuple             =	remove_space
   _a  :       Tuple             =	keep_accents
   _a  :       Tuple             =	vocab_file

   _a  :       Any             =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
   self.sp_model.Load(UpperCAmelCase__							)

   try:
     import jieba
   except ModuleNotFoundError as error:
     raise error.__class__(
         """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
         """See https://pypi.org/project/jieba/ for installation."""							)
   _a  :       int             =	jieba
   _a  :       Tuple             =	str.maketrans(""" \n"""							,						"""\u2582\u2583"""							)


 @property
 # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
 def _lowercase	(     self		:						Optional[Any]							)					->       Any:
   return len(self.sp_model							)


 def _lowercase	(     self		:						str							)					->       Union[str, Any]:
   _a  :       int             =	{self.convert_ids_to_tokens(UpperCAmelCase__							): i for i in range(self.vocab_size							)}
   vocab.update(self.added_tokens_encoder							)
   return vocab


 def __getstate__(     self		:						Tuple							)					->       List[str]:
   _a  :       Tuple             =	self.__dict__.copy()
   _a  :       Tuple             =	None
   return state


 def __setstate__(     self		:						Any							,						UpperCAmelCase__		:						Dict							)					->       Dict:
   _a  :       Tuple             =	d

   # for backward compatibility
   if not hasattr(self							,						"""sp_model_kwargs"""							):
     _a  :       Tuple             =	{}

   _a  :       int             =	spm.SentencePieceProcessor(**self.sp_model_kwargs							)
   self.sp_model.Load(self.vocab_file							)


 def _lowercase	(     self		:						List[str]							,						UpperCAmelCase__		:						Union[str, Any]							)					->       Dict:
   if self.remove_space:
     _a  :       Optional[int]             =	""" """.join(inputs.strip().split()							)
   else:
     _a  :       List[Any]             =	inputs
   _a  :       int             =	outputs.replace("""``"""							,						"""\""""							).replace("""''"""							,						"""\""""							)

   if not self.keep_accents:
     _a  :       Optional[Any]             =	unicodedata.normalize("""NFKD"""							,						UpperCAmelCase__							)
     _a  :       Dict             =	"""""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__							)]							)
   if self.do_lower_case:
     _a  :       Union[str, Any]             =	outputs.lower()

   return outputs


 def _lowercase	(     self		:						List[str]							,						UpperCAmelCase__		:						str							)					->       List[str]:
   _a  :       str             =	self.preprocess_text(UpperCAmelCase__							)
   _a  :       Dict             =	self.sp_model.encode(UpperCAmelCase__							,						out_type=UpperCAmelCase__							)
   _a  :       Union[str, Any]             =	[]
   for piece in pieces:
     if len(UpperCAmelCase__							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
       _a  :       Dict             =	self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__							,						""""""							)							)
       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]							) == 1:
           _a  :       Dict             =	cur_pieces[1:]
         else:
           _a  :       Any             =	cur_pieces[0][1:]
       cur_pieces.append(piece[-1]							)
       new_pieces.extend(UpperCAmelCase__							)
     else:
       new_pieces.append(UpperCAmelCase__							)

   return new_pieces


 def _lowercase	(     self		:						Union[str, Any]							,						UpperCAmelCase__		:						int							)					->       int:
   return self.sp_model.PieceToId(UpperCAmelCase__							)


 def _lowercase	(     self		:						List[Any]							,						UpperCAmelCase__		:						Optional[Any]							)					->       Any:
   return self.sp_model.IdToPiece(UpperCAmelCase__							)


 def _lowercase	(     self		:						Any							,						UpperCAmelCase__		:						Any							)					->       Dict:
   _a  :       Dict             =	"""""".join(UpperCAmelCase__							).replace(UpperCAmelCase__							,						""" """							).strip()
   return out_string


 def _lowercase	(     self		:						Any							,						UpperCAmelCase__		:						List[int]							,						UpperCAmelCase__		:						Optional[List[int]] = None							)					->       List[int]:
   _a  :       Optional[Any]             =	[self.sep_token_id]
   _a  :       Dict             =	[self.cls_token_id]
   if token_ids_a is None:
     return token_ids_a + sep + cls
   return token_ids_a + sep + token_ids_a + sep + cls


 def _lowercase	(     self		:						Tuple							,						UpperCAmelCase__		:						List[int]							,						UpperCAmelCase__		:						Optional[List[int]] = None							,						UpperCAmelCase__		:						bool = False							)					->       List[int]:

   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=UpperCAmelCase__							,						token_ids_a=UpperCAmelCase__							,						already_has_special_tokens=UpperCAmelCase__							)

   if token_ids_a is not None:
     return ([0] * len(UpperCAmelCase__							)) + [1] + ([0] * len(UpperCAmelCase__							)) + [1, 1]
   return ([0] * len(UpperCAmelCase__							)) + [1, 1]


 def _lowercase	(     self		:						str							,						UpperCAmelCase__		:						List[int]							,						UpperCAmelCase__		:						Optional[List[int]] = None							)					->       List[int]:
   _a  :       Any             =	[self.sep_token_id]
   _a  :       Optional[Any]             =	[2]

   if token_ids_a is None:
     return len(token_ids_a + sep							) * [0] + cls_segment_id
   return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id


 def _lowercase	(     self		:						Optional[int]							,						UpperCAmelCase__		:						str							,						UpperCAmelCase__		:						Optional[str] = None							)					->       Tuple[str]:
   if not os.path.isdir(UpperCAmelCase__							):
     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
     return
   _a  :       Union[str, Any]             =	os.path.join(
       UpperCAmelCase__							,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

   if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase__							) and os.path.isfile(self.vocab_file							):
     copyfile(self.vocab_file							,						UpperCAmelCase__							)
   elif not os.path.isfile(self.vocab_file							):
     with open(UpperCAmelCase__							,						"""wb"""							) as fi:
       _a  :       Optional[Any]             =	self.sp_model.serialized_model_proto()
       fi.write(UpperCAmelCase__							)

   return (out_vocab_file,)







 def _lowercase	(     self		:						Any							,						*UpperCAmelCase__		:						List[str]							,						**UpperCAmelCase__		:						List[str]							)					->       List[str]:
   _a  :       Tuple             =	super()._decode(*UpperCAmelCase__							,						**UpperCAmelCase__							)
   _a  :       Optional[Any]             =	text.replace(""" """							,						""""""							).replace("""\u2582"""							,						""" """							).replace("""\u2583"""							,						"""\n"""							)
   return text

