


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A       :		int    =   logging.get_logger(__name__)

A       :		int    =   {"vocab_file": "spiece.model"}

A       :		Union[str, Any]    =   {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

A       :		Optional[Any]    =   {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
A       :		Optional[int]    =   0
A       :		Optional[Any]    =   1
A       :		Union[str, Any]    =   2
A       :		Dict    =   3
A       :		List[Any]    =   4



class 							A    (       __lowercase							):
					'''simple docstring'''






					A__			= VOCAB_FILES_NAMES
					A__			= PRETRAINED_VOCAB_FILES_MAP
					A__			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__			= """left"""

					def __init__(self    :						List[str]      ,					_UpperCAmelCase    :						Dict      ,					_UpperCAmelCase    :						Optional[int]=False      ,					_UpperCAmelCase    :						Tuple=True      ,					_UpperCAmelCase    :						str=False      ,					_UpperCAmelCase    :						Dict="<s>"      ,					_UpperCAmelCase    :						int="</s>"      ,					_UpperCAmelCase    :						Dict="<unk>"      ,					_UpperCAmelCase    :						int="<sep>"      ,					_UpperCAmelCase    :						Tuple="<pad>"      ,					_UpperCAmelCase    :						List[str]="<cls>"      ,					_UpperCAmelCase    :						int="<mask>"      ,					_UpperCAmelCase    :						Dict=["<eop>", "<eod>"]      ,					_UpperCAmelCase    :						Optional[Dict[str, Any]] = None      ,					**_UpperCAmelCase    :						Optional[int]      ,					)						->   None:




							"""simple docstring"""


							lowercase__     			=		AddedToken(SCREAMING_SNAKE_CASE__      ,					lstrip=SCREAMING_SNAKE_CASE__      ,					rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__      ,					SCREAMING_SNAKE_CASE__		) else mask_token

							lowercase__     			=		{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=SCREAMING_SNAKE_CASE__      ,					remove_space=SCREAMING_SNAKE_CASE__      ,					keep_accents=SCREAMING_SNAKE_CASE__      ,					bos_token=SCREAMING_SNAKE_CASE__      ,					eos_token=SCREAMING_SNAKE_CASE__      ,					unk_token=SCREAMING_SNAKE_CASE__      ,					sep_token=SCREAMING_SNAKE_CASE__      ,					pad_token=SCREAMING_SNAKE_CASE__      ,					cls_token=SCREAMING_SNAKE_CASE__      ,					mask_token=SCREAMING_SNAKE_CASE__      ,					additional_special_tokens=SCREAMING_SNAKE_CASE__      ,					sp_model_kwargs=self.sp_model_kwargs      ,					**SCREAMING_SNAKE_CASE__      ,					)

							lowercase__     			=		3

							lowercase__     			=		do_lower_case
							lowercase__     			=		remove_space
							lowercase__     			=		keep_accents
							lowercase__     			=		vocab_file

							lowercase__     			=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
							self.sp_model.Load(SCREAMING_SNAKE_CASE__		)

					@property
					def   lowerCamelCase__     (self    :						Optional[int]		)						->   Dict:




							"""simple docstring"""


							return len(self.sp_model		)

					def   lowerCamelCase__     (self    :						Tuple		)						->   Tuple:




							"""simple docstring"""


							lowercase__     			=		{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__		): i for i in range(self.vocab_size		)}
							vocab.update(self.added_tokens_encoder		)
							return vocab

					def __getstate__(self    :						int		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		self.__dict__.copy()
							lowercase__     			=		None
							return state

					def __setstate__(self    :						str      ,					_UpperCAmelCase    :						Tuple		)						->   int:




							"""simple docstring"""


							lowercase__     			=		d

							# for backward compatibility
							if not hasattr(self      ,					"""sp_model_kwargs"""		):
									lowercase__     			=		{}

							lowercase__     			=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
							self.sp_model.Load(self.vocab_file		)

					def   lowerCamelCase__     (self    :						Tuple      ,					_UpperCAmelCase    :						int		)						->   List[Any]:




							"""simple docstring"""


							if self.remove_space:
									lowercase__     			=		""" """.join(inputs.strip().split()		)
							else:
									lowercase__     			=		inputs
							lowercase__     			=		outputs.replace("""``"""      ,					"""\""""		).replace("""\'\'"""      ,					"""\""""		)

							if not self.keep_accents:
									lowercase__     			=		unicodedata.normalize("""NFKD"""      ,					SCREAMING_SNAKE_CASE__		)
									lowercase__     			=		"""""".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE__		)]		)
							if self.do_lower_case:
									lowercase__     			=		outputs.lower()

							return outputs

					def   lowerCamelCase__     (self    :						Optional[Any]      ,					_UpperCAmelCase    :						str		)						->   List[str]:




							"""simple docstring"""


							lowercase__     			=		self.preprocess_text(SCREAMING_SNAKE_CASE__		)
							lowercase__     			=		self.sp_model.encode(SCREAMING_SNAKE_CASE__      ,					out_type=SCREAMING_SNAKE_CASE__		)
							lowercase__     			=		[]
							for piece in pieces:
									if len(SCREAMING_SNAKE_CASE__		) > 1 and piece[-1] == str(""","""		) and piece[-2].isdigit():
											lowercase__     			=		self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE__      ,					""""""		)		)
											if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
													if len(cur_pieces[0]		) == 1:
															lowercase__     			=		cur_pieces[1:]
													else:
															lowercase__     			=		cur_pieces[0][1:]
											cur_pieces.append(piece[-1]		)
											new_pieces.extend(SCREAMING_SNAKE_CASE__		)
									else:
											new_pieces.append(SCREAMING_SNAKE_CASE__		)

							return new_pieces

					def   lowerCamelCase__     (self    :						Dict      ,					_UpperCAmelCase    :						str		)						->   Optional[int]:




							"""simple docstring"""


							return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__		)

					def   lowerCamelCase__     (self    :						Union[str, Any]      ,					_UpperCAmelCase    :						Tuple		)						->   str:




							"""simple docstring"""


							return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE__		)

					def   lowerCamelCase__     (self    :						str      ,					_UpperCAmelCase    :						str		)						->   Union[str, Any]:




							"""simple docstring"""


							lowercase__     			=		"""""".join(SCREAMING_SNAKE_CASE__		).replace(SCREAMING_SNAKE_CASE__      ,					""" """		).strip()
							return out_string

					def   lowerCamelCase__     (self    :						Optional[int]      ,					_UpperCAmelCase    :						List[int]      ,					_UpperCAmelCase    :						bool = False      ,					_UpperCAmelCase    :						bool = None      ,					_UpperCAmelCase    :						bool = True      ,					**_UpperCAmelCase    :						List[Any]      ,					)						->   str:




							"""simple docstring"""


							lowercase__     			=		kwargs.pop("""use_source_tokenizer"""      ,					SCREAMING_SNAKE_CASE__		)

							lowercase__     			=		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__      ,					skip_special_tokens=SCREAMING_SNAKE_CASE__		)

							# To avoid mixing byte-level and unicode for byte-level BPT
							# we need to build string separately for added tokens and byte-level tokens
							# cf. https://github.com/huggingface/transformers/issues/1133
							lowercase__     			=		[]
							lowercase__     			=		[]
							for token in filtered_tokens:
									if skip_special_tokens and token in self.all_special_ids:
											continue
									if token in self.added_tokens_encoder:
											if current_sub_text:
													sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE__		)		)
													lowercase__     			=		[]
											sub_texts.append(SCREAMING_SNAKE_CASE__		)
									else:
											current_sub_text.append(SCREAMING_SNAKE_CASE__		)
							if current_sub_text:
									sub_texts.append(self.convert_tokens_to_string(SCREAMING_SNAKE_CASE__		)		)

							# Mimic the behavior of the Rust tokenizer:
							# By default, there are no spaces between special tokens
							lowercase__     			=		"""""".join(SCREAMING_SNAKE_CASE__		)

							lowercase__     			=		(
							    clean_up_tokenization_spaces
							    if clean_up_tokenization_spaces is not None
							    else self.clean_up_tokenization_spaces
							)
							if clean_up_tokenization_spaces:
									lowercase__     			=		self.clean_up_tokenization(SCREAMING_SNAKE_CASE__		)
									return clean_text
							else:
									return text

					def   lowerCamelCase__     (self    :						Optional[int]      ,					_UpperCAmelCase    :						List[int]      ,					_UpperCAmelCase    :						Optional[List[int]] = None		)						->   List[int]:




							"""simple docstring"""


							lowercase__     			=		[self.sep_token_id]
							lowercase__     			=		[self.cls_token_id]
							if token_ids_a is None:
									return token_ids_a + sep + cls
							return token_ids_a + sep + token_ids_a + sep + cls

					def   lowerCamelCase__     (self    :						Union[str, Any]      ,					_UpperCAmelCase    :						List[int]      ,					_UpperCAmelCase    :						Optional[List[int]] = None      ,					_UpperCAmelCase    :						bool = False		)						->   List[int]:




							"""simple docstring"""


							if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=SCREAMING_SNAKE_CASE__      ,					token_ids_a=SCREAMING_SNAKE_CASE__      ,					already_has_special_tokens=SCREAMING_SNAKE_CASE__		)

							if token_ids_a is not None:
									return ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1, 1]
							return ([0] * len(SCREAMING_SNAKE_CASE__		)) + [1, 1]

					def   lowerCamelCase__     (self    :						Tuple      ,					_UpperCAmelCase    :						List[int]      ,					_UpperCAmelCase    :						Optional[List[int]] = None		)						->   List[int]:




							"""simple docstring"""


							lowercase__     			=		[self.sep_token_id]
							lowercase__     			=		[2]

							if token_ids_a is None:
									return len(token_ids_a + sep		) * [0] + cls_segment_id
							return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id


					def   lowerCamelCase__     (self    :						Optional[Any]      ,					_UpperCAmelCase    :						str      ,					_UpperCAmelCase    :						Optional[str] = None		)						->   Tuple[str]:




							"""simple docstring"""


							if not os.path.isdir(SCREAMING_SNAKE_CASE__		):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
									return
							lowercase__     			=		os.path.join(
							    SCREAMING_SNAKE_CASE__      ,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

							if os.path.abspath(self.vocab_file		) != os.path.abspath(SCREAMING_SNAKE_CASE__		) and os.path.isfile(self.vocab_file		):
									copyfile(self.vocab_file      ,					SCREAMING_SNAKE_CASE__		)
							elif not os.path.isfile(self.vocab_file		):
									with open(SCREAMING_SNAKE_CASE__      ,					"""wb"""		) as fi:
											lowercase__     			=		self.sp_model.serialized_model_proto()
											fi.write(SCREAMING_SNAKE_CASE__		)

							return (out_vocab_file,)



from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
					if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")):
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
					from .pipeline_unclip import UnCLIPPipeline
					from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
					from .text_proj import UnCLIPTextProjModel



