


'''simple docstring'''



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase	:						str       =	logging.get_logger(__name__)
__lowercase	:						int       =	{'vocab_file': 'spiece.model'}

__lowercase	:						int       =	{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__lowercase	:						Union[str, Any]       =	{
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

__lowercase	:						Any       =	'▁'





class 							__UpperCamelCase				(		lowerCAmelCase_					):
	A_				=    VOCAB_FILES_NAMES
	A_				=    PRETRAINED_VOCAB_FILES_MAP
	A_				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




	def __init__(   self						,				__a						,				__a=True						,				__a=True						,				__a=False						,				__a="[CLS]"						,				__a="[SEP]"						,				__a="<unk>"						,				__a="[SEP]"						,				__a="<pad>"						,				__a="[CLS]"						,				__a="[MASK]"						,				__a = None						,				**__a						,				):

						'''simple docstring'''





						__a			:       List[str]					    =  (
						    AddedToken(__a						,				lstrip=__a						,				rstrip=__a						,				normalized=__a							)
						    if isinstance(__a						,				__a							)
						    else mask_token
						)

						__a			:       Dict					    =  {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__a						,				remove_space=__a						,				keep_accents=__a						,				bos_token=__a						,				eos_token=__a						,				unk_token=__a						,				sep_token=__a						,				pad_token=__a						,				cls_token=__a						,				mask_token=__a						,				sp_model_kwargs=self.sp_model_kwargs						,				**__a						,				)

						__a			:       Any					    =  do_lower_case
						__a			:       Any					    =  remove_space
						__a			:       Optional[Any]					    =  keep_accents
						__a			:       Union[str, Any]					    =  vocab_file

						__a			:       Any					    =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(__a							)




	@property
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						return len(self.sp_model							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  {self.convert_ids_to_tokens(__a							): i for i in range(self.vocab_size							)}
						vocab.update(self.added_tokens_encoder							)
						return vocab




	def __getstate__(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  self.__dict__.copy()
						__a			:       List[str]					    =  None
						return state




	def __setstate__(   self						,				__a							):

						'''simple docstring'''





						__a			:       Optional[Any]					    =  d

						# for backward compatibility
						if not hasattr(self						,				'sp_model_kwargs'							):
											__a			:       Optional[Any]					    =  {}

						__a			:       Union[str, Any]					    =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(self.vocab_file							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						if self.remove_space:
											__a			:       Union[str, Any]					    =  ' '.join(inputs.strip().split()							)
						else:
											__a			:       Dict					    =  inputs
						__a			:       Optional[Any]					    =  outputs.replace('``'						,				'"'							).replace('\'\''						,				'"'							)

						if not self.keep_accents:
											__a			:       Optional[Any]					    =  unicodedata.normalize('NFKD'						,				__a							)
											__a			:       List[str]					    =  ''.join([c for c in outputs if not unicodedata.combining(__a							)]							)
						if self.do_lower_case:
											__a			:       Tuple					    =  outputs.lower()

						return outputs




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						__a			:       Dict					    =  self.preprocess_text(__a							)
						__a			:       Tuple					    =  self.sp_model.encode(__a						,				out_type=__a							)
						__a			:       List[Any]					    =  []
						for piece in pieces:
											if len(__a							) > 1 and piece[-1] == str(','							) and piece[-2].isdigit():
																__a			:       List[str]					    =  self.sp_model.EncodeAsPieces(piece[:-1].replace(__a						,				''							)							)
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																					if len(cur_pieces[0]							) == 1:
																										__a			:       int					    =  cur_pieces[1:]
																					else:
																										__a			:       str					    =  cur_pieces[0][1:]
																cur_pieces.append(piece[-1]							)
																new_pieces.extend(__a							)
											else:
																new_pieces.append(__a							)

						return new_pieces




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						return self.sp_model.PieceToId(__a							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						return self.sp_model.IdToPiece(__a							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  []
						__a			:       List[Any]					    =  ''
						__a			:       int					    =  False
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																if not prev_is_special:
																					out_string += " "
																out_string += self.sp_model.decode(__a							) + token
																__a			:       List[Any]					    =  True
																__a			:       Union[str, Any]					    =  []
											else:
																current_sub_tokens.append(__a							)
																__a			:       Union[str, Any]					    =  False
						out_string += self.sp_model.decode(__a							)
						return out_string.strip()




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  [self.sep_token_id]
						__a			:       Optional[int]					    =  [self.cls_token_id]
						if token_ids_a is None:
											return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None						,				__a = False							):

						'''simple docstring'''





						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__a						,				token_ids_a=__a						,				already_has_special_tokens=__a							)

						if token_ids_a is not None:
											return [1] + ([0] * len(__a							)) + [1] + ([0] * len(__a							)) + [1]
						return [1] + ([0] * len(__a							)) + [1]




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						__a			:       Dict					    =  [self.sep_token_id]
						__a			:       Dict					    =  [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						if not os.path.isdir(__a							):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
											return
						__a			:       Tuple					    =  os.path.join(
						    __a						,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

						if os.path.abspath(self.vocab_file							) != os.path.abspath(__a							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file						,				__a							)
						elif not os.path.isfile(self.vocab_file							):
											with open(__a						,				'wb'							) as fi:
																__a			:       int					    =  self.sp_model.serialized_model_proto()
																fi.write(__a							)

						return (out_vocab_file,)








'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__lowerCAmelCase									=   {'''configuration_swin''': ['''SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''SwinConfig''', '''SwinOnnxConfig''']}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase									=   [
							    '''SWIN_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''SwinForImageClassification''',
							    '''SwinForMaskedImageModeling''',
							    '''SwinModel''',
							    '''SwinPreTrainedModel''',
							    '''SwinBackbone''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase									=   [
							    '''TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFSwinForImageClassification''',
							    '''TFSwinForMaskedImageModeling''',
							    '''TFSwinModel''',
							    '''TFSwinPreTrainedModel''',
							]

if TYPE_CHECKING:
							from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_swin import (
														    SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
														    SwinBackbone,
														    SwinForImageClassification,
														    SwinForMaskedImageModeling,
														    SwinModel,
														    SwinPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_swin import (
														    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFSwinForImageClassification,
														    TFSwinForMaskedImageModeling,
														    TFSwinModel,
														    TFSwinPreTrainedModel,
														)

else:
							import sys

							__lowerCAmelCase									=   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




