




"""simple docstring"""

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase     :	Dict         =				logging.get_logger(__name__)
_lowerCamelCase     :	Optional[Any]         =				{"""vocab_file""": """spiece.model"""}

_lowerCamelCase     :	Tuple         =				{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

_lowerCamelCase     :	int         =				{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

_lowerCamelCase     :	List[Any]         =				"""▁"""

class 				lowercase					(			UpperCamelCase__):
				__lowerCAmelCase			:			List[Any]     				=      VOCAB_FILES_NAMES
				__lowerCAmelCase			:			int     				=      PRETRAINED_VOCAB_FILES_MAP
				__lowerCAmelCase			:			Dict     				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







				def __init__(							self      :    Any					,   _lowerCamelCase      :    List[str]					,   _lowerCamelCase      :    List[Any]=True					,   _lowerCamelCase      :    int=True					,   _lowerCamelCase      :    List[Any]=False					,   _lowerCamelCase      :    Union[str, Any]="[CLS]"					,   _lowerCamelCase      :    str="[SEP]"					,   _lowerCamelCase      :    List[Any]="<unk>"					,   _lowerCamelCase      :    Optional[Any]="[SEP]"					,   _lowerCamelCase      :    List[str]="<pad>"					,   _lowerCamelCase      :    Optional[Any]="[CLS]"					,   _lowerCamelCase      :    List[Any]="[MASK]"					,   _lowerCamelCase      :    Optional[int] = None					,   **_lowerCamelCase      :    Any					,   ):



					"""simple docstring"""

					A_     : List[str]		      =						(
					    AddedToken(lowerCAmelCase__					,   lstrip=lowerCAmelCase__					,   rstrip=lowerCAmelCase__					,   normalized=lowerCAmelCase__				)
					    if isinstance(lowerCAmelCase__					,   lowerCAmelCase__				)
					    else mask_token
					)

					A_     : Dict		      =						{} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    do_lower_case=lowerCAmelCase__					,   remove_space=lowerCAmelCase__					,   keep_accents=lowerCAmelCase__					,   bos_token=lowerCAmelCase__					,   eos_token=lowerCAmelCase__					,   unk_token=lowerCAmelCase__					,   sep_token=lowerCAmelCase__					,   pad_token=lowerCAmelCase__					,   cls_token=lowerCAmelCase__					,   mask_token=lowerCAmelCase__					,   sp_model_kwargs=self.sp_model_kwargs					,   **lowerCAmelCase__					,   )

					A_     : Tuple		      =						do_lower_case
					A_     : int		      =						remove_space
					A_     : List[Any]		      =						keep_accents
					A_     : List[Any]		      =						vocab_file

					A_     : List[str]		      =						spm.SentencePieceProcessor(**self.sp_model_kwargs				)
					self.sp_model.Load(lowerCAmelCase__				)







				@property
				def     a_     (							self      :    Optional[int]				):



					"""simple docstring"""

					return len(self.sp_model				)







				def     a_     (							self      :    Optional[int]				):



					"""simple docstring"""

					A_     : Tuple		      =						{self.convert_ids_to_tokens(lowerCAmelCase__				): i for i in range(self.vocab_size				)}
					vocab.update(self.added_tokens_encoder				)
					return vocab







				def __getstate__(							self      :    Any				):



					"""simple docstring"""

					A_     : Union[str, Any]		      =						self.__dict__.copy()
					A_     : Optional[int]		      =						None
					return state







				def __setstate__(							self      :    Dict					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					A_     : List[Any]		      =						d

					# for backward compatibility
					if not hasattr(self					,   '''sp_model_kwargs'''				):
						A_     : Optional[int]		      =						{}

					A_     : str		      =						spm.SentencePieceProcessor(**self.sp_model_kwargs				)
					self.sp_model.Load(self.vocab_file				)







				def     a_     (							self      :    List[Any]					,   _lowerCamelCase      :    List[str]				):



					"""simple docstring"""

					if self.remove_space:
						A_     : Dict		      =						" ".join(inputs.strip().split()				)
					else:
						A_     : List[Any]		      =						inputs
					A_     : str		      =						outputs.replace('''``'''					,   '''\"'''				).replace('''\'\''''					,   '''\"'''				)

					if not self.keep_accents:
						A_     : List[Any]		      =						unicodedata.normalize('''NFKD'''					,   lowerCAmelCase__				)
						A_     : Any		      =						"".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__				)]				)
					if self.do_lower_case:
						A_     : int		      =						outputs.lower()

					return outputs







				def     a_     (							self      :    Optional[int]					,   _lowerCamelCase      :    List[Any]				):



					"""simple docstring"""

					A_     : Optional[int]		      =						self.preprocess_text(lowerCAmelCase__				)
					A_     : Optional[Any]		      =						self.sp_model.encode(lowerCAmelCase__					,   out_type=lowerCAmelCase__				)
					A_     : Optional[int]		      =						[]
					for piece in pieces:
						if len(lowerCAmelCase__				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
							A_     : Optional[int]		      =						self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__					,   ''''''				)				)
							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
								if len(cur_pieces[0]				) == 1:
									A_     : Optional[Any]		      =						cur_pieces[1:]
								else:
									A_     : List[str]		      =						cur_pieces[0][1:]
							cur_pieces.append(piece[-1]				)
							new_pieces.extend(lowerCAmelCase__				)
						else:
							new_pieces.append(lowerCAmelCase__				)

					return new_pieces







				def     a_     (							self      :    str					,   _lowerCamelCase      :    Optional[int]				):



					"""simple docstring"""

					return self.sp_model.PieceToId(lowerCAmelCase__				)







				def     a_     (							self      :    Dict					,   _lowerCamelCase      :    Any				):



					"""simple docstring"""

					return self.sp_model.IdToPiece(lowerCAmelCase__				)







				def     a_     (							self      :    Tuple					,   _lowerCamelCase      :    Tuple				):



					"""simple docstring"""

					A_     : Dict		      =						[]
					A_     : Tuple		      =						""
					A_     : Tuple		      =						False
					for token in tokens:
						# make sure that special tokens are not decoded using sentencepiece model
						if token in self.all_special_tokens:
							if not prev_is_special:
								out_string += " "
							out_string += self.sp_model.decode(lowerCAmelCase__				) + token
							A_     : str		      =						True
							A_     : Tuple		      =						[]
						else:
							current_sub_tokens.append(lowerCAmelCase__				)
							A_     : List[Any]		      =						False
					out_string += self.sp_model.decode(lowerCAmelCase__				)
					return out_string.strip()







				def     a_     (							self      :    Tuple					,   _lowerCamelCase      :    Union[str, Any]					,   _lowerCamelCase      :    Optional[Any] = None				):



					"""simple docstring"""

					A_     : Union[str, Any]		      =						[self.sep_token_id]
					A_     : List[Any]		      =						[self.cls_token_id]
					if token_ids_a is None:
						return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep







				def     a_     (							self      :    Dict					,   _lowerCamelCase      :    Optional[Any]					,   _lowerCamelCase      :    Union[str, Any] = None					,   _lowerCamelCase      :    int = False				):



					"""simple docstring"""

					if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=lowerCAmelCase__					,   token_ids_a=lowerCAmelCase__					,   already_has_special_tokens=lowerCAmelCase__				)

					if token_ids_a is not None:
						return [1] + ([0] * len(lowerCAmelCase__				)) + [1] + ([0] * len(lowerCAmelCase__				)) + [1]
					return [1] + ([0] * len(lowerCAmelCase__				)) + [1]







				def     a_     (							self      :    Optional[Any]					,   _lowerCamelCase      :    int					,   _lowerCamelCase      :    int = None				):



					"""simple docstring"""

					A_     : Tuple		      =						[self.sep_token_id]
					A_     : Union[str, Any]		      =						[self.cls_token_id]

					if token_ids_a is None:
						return len(cls + token_ids_a + sep				) * [0]
					return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]







				def     a_     (							self      :    Optional[int]					,   _lowerCamelCase      :    List[Any]					,   _lowerCamelCase      :    int = None				):



					"""simple docstring"""

					if not os.path.isdir(lowerCAmelCase__				):
						logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
						return
					A_     : List[str]		      =						os.path.join(
					    lowerCAmelCase__					,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

					if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCAmelCase__				) and os.path.isfile(self.vocab_file				):
						copyfile(self.vocab_file					,   lowerCAmelCase__				)
					elif not os.path.isfile(self.vocab_file				):
						with open(lowerCAmelCase__					,   '''wb'''				) as fi:
							A_     : List[str]		      =						self.sp_model.serialized_model_proto()
							fi.write(lowerCAmelCase__				)

					return (out_vocab_file,)












import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import MobileNetVaImageProcessor







class 				__lowerCAmelCase       (    unittest.TestCase):







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__=7		,      lowerCAmelCase__=3		,      lowerCAmelCase__=1_8		,      lowerCAmelCase__=3_0		,      lowerCAmelCase__=4_0_0		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      )   ->						Optional[int]:



        '''simple docstring'''




        a__	:					str										=size if size is not None else {"shortest_edge": 2_0}
        a__	:					Union[str, Any]										=crop_size if crop_size is not None else {"height": 1_8, "width": 1_8}
        a__	:					Tuple										=parent
        a__	:					Optional[int]										=batch_size
        a__	:					Any										=num_channels
        a__	:					List[str]										=image_size
        a__	:					Dict										=min_resolution
        a__	:					List[Any]										=max_resolution
        a__	:					Dict										=do_resize
        a__	:					Union[str, Any]										=size
        a__	:					str										=do_center_crop
        a__	:					List[str]										=crop_size







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        return {
            "do_resize": self.do_resize,
            "size": self.size,
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
        }







@require_torch
@require_vision
class 				__lowerCAmelCase       (    UpperCamelCase__						,				unittest.TestCase):
      _lowercase				:					Optional[Any]						=			MobileNetVaImageProcessor if is_vision_available() else None







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__	:					Optional[int]										=MobileNetVaImageProcessingTester(self							)







      @property
      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        return self.image_processor_tester.prepare_image_processor_dict()







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					List[str]										=self.image_processing_class(**self.image_processor_dict							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "do_resize"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "size"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "do_center_crop"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "crop_size"							)							)







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					Any										=self.image_processing_class.from_dict(self.image_processor_dict							)
        self.assertEqual(image_processor.size		,      {"shortest_edge": 2_0}							)
        self.assertEqual(image_processor.crop_size		,      {"height": 1_8, "width": 1_8}							)

        a__	:					Dict										=self.image_processing_class.from_dict(self.image_processor_dict		,      size=4_2		,      crop_size=8_4							)
        self.assertEqual(image_processor.size		,      {"shortest_edge": 4_2}							)
        self.assertEqual(image_processor.crop_size		,      {"height": 8_4, "width": 8_4}							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        pass







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Dict										=self.image_processing_class(**self.image_processor_dict							)
        # create random PIL images
        a__	:					Optional[Any]										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      Image.Image							)

        # Test not batched input
        a__	:					List[Any]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					Dict										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        a__	:					str										=self.image_processing_class(**self.image_processor_dict							)
        # create random numpy tensors
        a__	:					str										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__		,      numpify=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      np.ndarray							)

        # Test not batched input
        a__	:					List[str]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					Union[str, Any]										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Any										=self.image_processing_class(**self.image_processor_dict							)
        # create random PyTorch tensors
        a__	:					int										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__		,      torchify=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      torch.Tensor							)

        # Test not batched input
        a__	:					Optional[Any]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					str										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )






