



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_    :  Optional[int]						  =		logging.get_logger(__name__)
a_    :  Any						  =		{'vocab_file': 'spiece.model'}

a_    :  Optional[int]						  =		{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

a_    :  int						  =		{
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

a_    :  Dict						  =		'▁'





class  lowerCamelCase__	(    UpperCAmelCase_):
     """simple docstring"""



     _A      =      VOCAB_FILES_NAMES
     _A      =      PRETRAINED_VOCAB_FILES_MAP
     _A      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







     def __init__(self					,      __a					,      __a=True					,      __a=True					,      __a=False					,      __a="[CLS]"					,      __a="[SEP]"					,      __a="<unk>"					,      __a="[SEP]"					,      __a="<pad>"					,      __a="[CLS]"					,      __a="[MASK]"					,      __a = None					,      **__a					,      ):






           '''simple docstring'''



           lowerCamelCase             =		(
               AddedToken(__a					,      lstrip=__a					,      rstrip=__a					,      normalized=__a     )
               if isinstance(__a					,      __a     )
               else mask_token
           )

           lowerCamelCase             =		{} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__a					,      remove_space=__a					,      keep_accents=__a					,      bos_token=__a					,      eos_token=__a					,      unk_token=__a					,      sep_token=__a					,      pad_token=__a					,      cls_token=__a					,      mask_token=__a					,      sp_model_kwargs=self.sp_model_kwargs					,      **__a					,      )

           lowerCamelCase             =		do_lower_case
           lowerCamelCase             =		remove_space
           lowerCamelCase             =		keep_accents
           lowerCamelCase             =		vocab_file

           lowerCamelCase             =		spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(__a     )







     @property
     def     _a   (self     ):






           '''simple docstring'''



           return len(self.sp_model     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		{self.convert_ids_to_tokens(__a     ): i for i in range(self.vocab_size     )}
           vocab.update(self.added_tokens_encoder     )
           return vocab







     def __getstate__(self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.__dict__.copy()
           lowerCamelCase             =		None
           return state







     def __setstate__(self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		d

           # for backward compatibility
           if not hasattr(self					,      "sp_model_kwargs"     ):
                 lowerCamelCase             =		{}

           lowerCamelCase             =		spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.vocab_file     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           if self.remove_space:
                 lowerCamelCase             =		" ".join(inputs.strip().split()     )
           else:
                 lowerCamelCase             =		inputs
           lowerCamelCase             =		outputs.replace("``"					,      "\""     ).replace("''"					,      "\""     )

           if not self.keep_accents:
                 lowerCamelCase             =		unicodedata.normalize("NFKD"					,      __a     )
                 lowerCamelCase             =		"".join([c for c in outputs if not unicodedata.combining(__a     )]     )
           if self.do_lower_case:
                 lowerCamelCase             =		outputs.lower()

           return outputs







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		self.preprocess_text(__a     )
           lowerCamelCase             =		self.sp_model.encode(__a					,      out_type=__a     )
           lowerCamelCase             =		[]
           for piece in pieces:
                 if len(__a     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
                       lowerCamelCase             =		self.sp_model.EncodeAsPieces(piece[:-1].replace(__a					,      ""     )     )
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                             if len(cur_pieces[0]     ) == 1:
                                   lowerCamelCase             =		cur_pieces[1:]
                             else:
                                   lowerCamelCase             =		cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]     )
                       new_pieces.extend(__a     )
                 else:
                       new_pieces.append(__a     )

           return new_pieces







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           return self.sp_model.PieceToId(__a     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           return self.sp_model.IdToPiece(__a     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		[]
           lowerCamelCase             =		""
           lowerCamelCase             =		False
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       if not prev_is_special:
                             out_string += " "
                       out_string += self.sp_model.decode(__a     ) + token
                       lowerCamelCase             =		True
                       lowerCamelCase             =		[]
                 else:
                       current_sub_tokens.append(__a     )
                       lowerCamelCase             =		False
           out_string += self.sp_model.decode(__a     )
           return out_string.strip()







     def     _a   (self					,      __a					,      __a = None     ):






           '''simple docstring'''



           lowerCamelCase             =		[self.sep_token_id]
           lowerCamelCase             =		[self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep







     def     _a   (self					,      __a					,      __a = None					,      __a = False     ):






           '''simple docstring'''



           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__a					,      token_ids_a=__a					,      already_has_special_tokens=__a     )

           if token_ids_a is not None:
                 return [1] + ([0] * len(__a     )) + [1] + ([0] * len(__a     )) + [1]
           return [1] + ([0] * len(__a     )) + [1]







     def     _a   (self					,      __a					,      __a = None     ):






           '''simple docstring'''



           lowerCamelCase             =		[self.sep_token_id]
           lowerCamelCase             =		[self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep     ) * [0]
           return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







     def     _a   (self					,      __a					,      __a = None     ):






           '''simple docstring'''



           if not os.path.isdir(__a     ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
                 return
           lowerCamelCase             =		os.path.join(
               __a					,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

           if os.path.abspath(self.vocab_file     ) != os.path.abspath(__a     ) and os.path.isfile(self.vocab_file     ):
                 copyfile(self.vocab_file					,      __a     )
           elif not os.path.isfile(self.vocab_file     ):
                 with open(__a					,      "wb"     ) as fi:
                       lowerCamelCase             =		self.sp_model.serialized_model_proto()
                       fi.write(__a     )

           return (out_vocab_file,)




import tempfile

import torch

from diffusers import PNDMScheduler

from .test_schedulers import SchedulerCommonTest





class  lowerCamelCase__	(    UpperCAmelCase_):
     """simple docstring"""



     _A      =      (PNDMScheduler,)
     _A      =      (('num_inference_steps', 50),)







     def     _a   (self					,      **__a     ):






           '''simple docstring'''



           lowerCamelCase             =		{
               "num_train_timesteps": 10_00,
               "beta_start": 0.0001,
               "beta_end": 0.02,
               "beta_schedule": "linear",
           }

           config.update(**__a     )
           return config







     def     _a   (self					,      __a=0					,      **__a     ):






           '''simple docstring'''



           lowerCamelCase             =		dict(self.forward_default_kwargs     )
           lowerCamelCase             =		kwargs.pop("num_inference_steps"					,      __a     )
           lowerCamelCase             =		self.dummy_sample
           lowerCamelCase             =		0.1 * sample
           lowerCamelCase             =		[residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]

           for scheduler_class in self.scheduler_classes:
                 lowerCamelCase             =		self.get_scheduler_config(**__a     )
                 lowerCamelCase             =		scheduler_class(**__a     )
                 scheduler.set_timesteps(__a     )
                 # copy over dummy past residuals
                 lowerCamelCase             =		dummy_past_residuals[:]

                 with tempfile.TemporaryDirectory() as tmpdirname:
                       scheduler.save_config(__a     )
                       lowerCamelCase             =		scheduler_class.from_pretrained(__a     )
                       new_scheduler.set_timesteps(__a     )
                       # copy over dummy past residuals
                       lowerCamelCase             =		dummy_past_residuals[:]

                 lowerCamelCase             =		scheduler.step_prk(__a					,      __a					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		new_scheduler.step_prk(__a					,      __a					,      __a					,      **__a     ).prev_sample

                 assert torch.sum(torch.abs(output - new_output     )     ) < 1E-5, "Scheduler outputs are not identical"

                 lowerCamelCase             =		scheduler.step_plms(__a					,      __a					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		new_scheduler.step_plms(__a					,      __a					,      __a					,      **__a     ).prev_sample

                 assert torch.sum(torch.abs(output - new_output     )     ) < 1E-5, "Scheduler outputs are not identical"







     def     _a   (self     ):






           '''simple docstring'''



           pass







     def     _a   (self					,      __a=0					,      **__a     ):






           '''simple docstring'''



           lowerCamelCase             =		dict(self.forward_default_kwargs     )
           lowerCamelCase             =		kwargs.pop("num_inference_steps"					,      __a     )
           lowerCamelCase             =		self.dummy_sample
           lowerCamelCase             =		0.1 * sample
           lowerCamelCase             =		[residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]

           for scheduler_class in self.scheduler_classes:
                 lowerCamelCase             =		self.get_scheduler_config()
                 lowerCamelCase             =		scheduler_class(**__a     )
                 scheduler.set_timesteps(__a     )

                 # copy over dummy past residuals (must be after setting timesteps)
                 lowerCamelCase             =		dummy_past_residuals[:]

                 with tempfile.TemporaryDirectory() as tmpdirname:
                       scheduler.save_config(__a     )
                       lowerCamelCase             =		scheduler_class.from_pretrained(__a     )
                       # copy over dummy past residuals
                       new_scheduler.set_timesteps(__a     )

                       # copy over dummy past residual (must be after setting timesteps)
                       lowerCamelCase             =		dummy_past_residuals[:]

                 lowerCamelCase             =		scheduler.step_prk(__a					,      __a					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		new_scheduler.step_prk(__a					,      __a					,      __a					,      **__a     ).prev_sample

                 assert torch.sum(torch.abs(output - new_output     )     ) < 1E-5, "Scheduler outputs are not identical"

                 lowerCamelCase             =		scheduler.step_plms(__a					,      __a					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		new_scheduler.step_plms(__a					,      __a					,      __a					,      **__a     ).prev_sample

                 assert torch.sum(torch.abs(output - new_output     )     ) < 1E-5, "Scheduler outputs are not identical"







     def     _a   (self					,      **__a     ):






           '''simple docstring'''



           lowerCamelCase             =		self.scheduler_classes[0]
           lowerCamelCase             =		self.get_scheduler_config(**__a     )
           lowerCamelCase             =		scheduler_class(**__a     )

           lowerCamelCase             =		10
           lowerCamelCase             =		self.dummy_model()
           lowerCamelCase             =		self.dummy_sample_deter
           scheduler.set_timesteps(__a     )

           for i, t in enumerate(scheduler.prk_timesteps     ):
                 lowerCamelCase             =		model(__a					,      __a     )
                 lowerCamelCase             =		scheduler.step_prk(__a					,      __a					,      __a     ).prev_sample

           for i, t in enumerate(scheduler.plms_timesteps     ):
                 lowerCamelCase             =		model(__a					,      __a     )
                 lowerCamelCase             =		scheduler.step_plms(__a					,      __a					,      __a     ).prev_sample

           return sample







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		dict(self.forward_default_kwargs     )

           lowerCamelCase             =		kwargs.pop("num_inference_steps"					,      __a     )

           for scheduler_class in self.scheduler_classes:
                 lowerCamelCase             =		self.get_scheduler_config()
                 lowerCamelCase             =		scheduler_class(**__a     )

                 lowerCamelCase             =		self.dummy_sample
                 lowerCamelCase             =		0.1 * sample

                 if num_inference_steps is not None and hasattr(__a					,      "set_timesteps"     ):
                       scheduler.set_timesteps(__a     )
                 elif num_inference_steps is not None and not hasattr(__a					,      "set_timesteps"     ):
                       lowerCamelCase             =		num_inference_steps

                 # copy over dummy past residuals (must be done after set_timesteps)
                 lowerCamelCase             =		[residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]
                 lowerCamelCase             =		dummy_past_residuals[:]

                 lowerCamelCase             =		scheduler.step_prk(__a					,      0					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		scheduler.step_prk(__a					,      1					,      __a					,      **__a     ).prev_sample

                 self.assertEqual(output_a.shape					,      sample.shape     )
                 self.assertEqual(output_a.shape					,      output_a.shape     )

                 lowerCamelCase             =		scheduler.step_plms(__a					,      0					,      __a					,      **__a     ).prev_sample
                 lowerCamelCase             =		scheduler.step_plms(__a					,      1					,      __a					,      **__a     ).prev_sample

                 self.assertEqual(output_a.shape					,      sample.shape     )
                 self.assertEqual(output_a.shape					,      output_a.shape     )







     def     _a   (self     ):






           '''simple docstring'''



           for timesteps in [1_00, 10_00]:
                 self.check_over_configs(num_train_timesteps=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           for steps_offset in [0, 1]:
                 self.check_over_configs(steps_offset=__a     )

           lowerCamelCase             =		self.scheduler_classes[0]
           lowerCamelCase             =		self.get_scheduler_config(steps_offset=1     )
           lowerCamelCase             =		scheduler_class(**__a     )
           scheduler.set_timesteps(10     )
           assert torch.equal(
               scheduler.timesteps					,      torch.LongTensor(
                   [9_01, 8_51, 8_51, 8_01, 8_01, 7_51, 7_51, 7_01, 7_01, 6_51, 6_51, 6_01, 6_01, 5_01, 4_01, 3_01, 2_01, 1_01, 1]     )					,      )







     def     _a   (self     ):






           '''simple docstring'''



           for beta_start, beta_end in zip([0.0001, 0.001]					,      [0.002, 0.02]     ):
                 self.check_over_configs(beta_start=__a					,      beta_end=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           for schedule in ["linear", "squaredcos_cap_v2"]:
                 self.check_over_configs(beta_schedule=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           for prediction_type in ["epsilon", "v_prediction"]:
                 self.check_over_configs(prediction_type=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           for t in [1, 5, 10]:
                 self.check_over_forward(time_step=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           for t, num_inference_steps in zip([1, 5, 10]					,      [10, 50, 1_00]     ):
                 self.check_over_forward(num_inference_steps=__a     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		27

           for scheduler_class in self.scheduler_classes:
                 lowerCamelCase             =		self.dummy_sample
                 lowerCamelCase             =		0.1 * sample

                 lowerCamelCase             =		self.get_scheduler_config()
                 lowerCamelCase             =		scheduler_class(**__a     )

                 scheduler.set_timesteps(__a     )

                 # before power of 3 fix, would error on first step, so we only need to do two
                 for i, t in enumerate(scheduler.prk_timesteps[:2]     ):
                       lowerCamelCase             =		scheduler.step_prk(__a					,      __a					,      __a     ).prev_sample







     def     _a   (self     ):






           '''simple docstring'''



           with self.assertRaises(__a     ):
                 lowerCamelCase             =		self.scheduler_classes[0]
                 lowerCamelCase             =		self.get_scheduler_config()
                 lowerCamelCase             =		scheduler_class(**__a     )

                 scheduler.step_plms(self.dummy_sample					,      1					,      self.dummy_sample     ).prev_sample







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.full_loop()
           lowerCamelCase             =		torch.sum(torch.abs(__a     )     )
           lowerCamelCase             =		torch.mean(torch.abs(__a     )     )

           assert abs(result_sum.item() - 198.1318     ) < 1E-2
           assert abs(result_mean.item() - 0.2580     ) < 1E-3







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.full_loop(prediction_type="v_prediction"     )
           lowerCamelCase             =		torch.sum(torch.abs(__a     )     )
           lowerCamelCase             =		torch.mean(torch.abs(__a     )     )

           assert abs(result_sum.item() - 67.3986     ) < 1E-2
           assert abs(result_mean.item() - 0.0878     ) < 1E-3







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.full_loop(set_alpha_to_one=__a					,      beta_start=0.01     )
           lowerCamelCase             =		torch.sum(torch.abs(__a     )     )
           lowerCamelCase             =		torch.mean(torch.abs(__a     )     )

           assert abs(result_sum.item() - 230.0399     ) < 1E-2
           assert abs(result_mean.item() - 0.2995     ) < 1E-3







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.full_loop(set_alpha_to_one=__a					,      beta_start=0.01     )
           lowerCamelCase             =		torch.sum(torch.abs(__a     )     )
           lowerCamelCase             =		torch.mean(torch.abs(__a     )     )

           assert abs(result_sum.item() - 186.9482     ) < 1E-2
           assert abs(result_mean.item() - 0.2434     ) < 1E-3