





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase	:	Dict		     =       logging.get_logger(__name__)
UpperCAmelCase	:	Optional[Any]		     =       {"""vocab_file""": """spiece.model"""}

UpperCAmelCase	:	Tuple		     =       {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

UpperCAmelCase	:	int		     =       {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

UpperCAmelCase	:	List[Any]		     =       """▁"""







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					List[Any]						=			VOCAB_FILES_NAMES
      _lowercase				:					int						=			PRETRAINED_VOCAB_FILES_MAP
      _lowercase				:					Dict						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__=True		,      lowerCAmelCase__=True		,      lowerCAmelCase__=False		,      lowerCAmelCase__="[CLS]"		,      lowerCAmelCase__="[SEP]"		,      lowerCAmelCase__="<unk>"		,      lowerCAmelCase__="[SEP]"		,      lowerCAmelCase__="<pad>"		,      lowerCAmelCase__="[CLS]"		,      lowerCAmelCase__="[MASK]"		,      lowerCAmelCase__ = None		,      **lowerCAmelCase__		,      )   ->						None:



        '''simple docstring'''




        a__	:					List[str]										=(
            AddedToken(lowerCAmelCase__		,      lstrip=lowerCAmelCase__		,      rstrip=lowerCAmelCase__		,      normalized=lowerCAmelCase__							)
            if isinstance(lowerCAmelCase__		,      lowerCAmelCase__							)
            else mask_token
        )

        a__	:					Dict										={} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            do_lower_case=lowerCAmelCase__		,      remove_space=lowerCAmelCase__		,      keep_accents=lowerCAmelCase__		,      bos_token=lowerCAmelCase__		,      eos_token=lowerCAmelCase__		,      unk_token=lowerCAmelCase__		,      sep_token=lowerCAmelCase__		,      pad_token=lowerCAmelCase__		,      cls_token=lowerCAmelCase__		,      mask_token=lowerCAmelCase__		,      sp_model_kwargs=self.sp_model_kwargs		,      **lowerCAmelCase__		,      )

        a__	:					Tuple										=do_lower_case
        a__	:					int										=remove_space
        a__	:					List[Any]										=keep_accents
        a__	:					List[Any]										=vocab_file

        a__	:					List[str]										=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
        self.sp_model.Load(lowerCAmelCase__							)







      @property
      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        return len(self.sp_model							)







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Tuple										={self.convert_ids_to_tokens(lowerCAmelCase__							): i for i in range(self.vocab_size							)}
        vocab.update(self.added_tokens_encoder							)
        return vocab







      def __getstate__(    self							)   ->						Union[str, Any]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=self.__dict__.copy()
        a__	:					Optional[int]										=None
        return state







      def __setstate__(    self		,      lowerCAmelCase__							)   ->						Dict:



        '''simple docstring'''




        a__	:					List[Any]										=d

        # for backward compatibility
        if not hasattr(self		,      "sp_model_kwargs"							):
          a__	:					Optional[int]										={}

        a__	:					str										=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
        self.sp_model.Load(self.vocab_file							)







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Optional[Any]:



        '''simple docstring'''




        if self.remove_space:
          a__	:					Dict										=" ".join(inputs.strip().split()							)
        else:
          a__	:					List[Any]										=inputs
        a__	:					str										=outputs.replace("``"		,      "\""							).replace("''"		,      "\""							)

        if not self.keep_accents:
          a__	:					List[Any]										=unicodedata.normalize("NFKD"		,      lowerCAmelCase__							)
          a__	:					Any										="".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__							)]							)
        if self.do_lower_case:
          a__	:					int										=outputs.lower()

        return outputs







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						List[str]:



        '''simple docstring'''




        a__	:					Optional[int]										=self.preprocess_text(lowerCAmelCase__							)
        a__	:					Optional[Any]										=self.sp_model.encode(lowerCAmelCase__		,      out_type=lowerCAmelCase__							)
        a__	:					Optional[int]										=[]
        for piece in pieces:
          if len(lowerCAmelCase__							) > 1 and piece[-1] == str(","							) and piece[-2].isdigit():
            a__	:					Optional[int]										=self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__		,      ""							)							)
            if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
              if len(cur_pieces[0]							) == 1:
                a__	:					Optional[Any]										=cur_pieces[1:]
              else:
                a__	:					List[str]										=cur_pieces[0][1:]
            cur_pieces.append(piece[-1]							)
            new_pieces.extend(lowerCAmelCase__							)
          else:
            new_pieces.append(lowerCAmelCase__							)

        return new_pieces







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						List[Any]:



        '''simple docstring'''




        return self.sp_model.PieceToId(lowerCAmelCase__							)







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Optional[int]:



        '''simple docstring'''




        return self.sp_model.IdToPiece(lowerCAmelCase__							)







      def    _lowercase  (    self		,      lowerCAmelCase__							)   ->						Tuple:



        '''simple docstring'''




        a__	:					Dict										=[]
        a__	:					Tuple										=""
        a__	:					Tuple										=False
        for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
            if not prev_is_special:
              out_string += " "
            out_string += self.sp_model.decode(lowerCAmelCase__							) + token
            a__	:					str										=True
            a__	:					Tuple										=[]
          else:
            current_sub_tokens.append(lowerCAmelCase__							)
            a__	:					List[Any]										=False
        out_string += self.sp_model.decode(lowerCAmelCase__							)
        return out_string.strip()







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						List[int]:



        '''simple docstring'''




        a__	:					Union[str, Any]										=[self.sep_token_id]
        a__	:					List[Any]										=[self.cls_token_id]
        if token_ids_a is None:
          return cls + token_ids_a + sep
        return cls + token_ids_a + sep + token_ids_a + sep







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = False							)   ->						List[int]:



        '''simple docstring'''




        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=lowerCAmelCase__		,      token_ids_a=lowerCAmelCase__		,      already_has_special_tokens=lowerCAmelCase__							)

        if token_ids_a is not None:
          return [1] + ([0] * len(lowerCAmelCase__							)) + [1] + ([0] * len(lowerCAmelCase__							)) + [1]
        return [1] + ([0] * len(lowerCAmelCase__							)) + [1]







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						List[int]:



        '''simple docstring'''




        a__	:					Tuple										=[self.sep_token_id]
        a__	:					Union[str, Any]										=[self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep							) * [0]
        return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						Tuple[str]:



        '''simple docstring'''




        if not os.path.isdir(lowerCAmelCase__							):
          logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
          return
        a__	:					List[str]										=os.path.join(
            lowerCAmelCase__		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

        if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCAmelCase__							) and os.path.isfile(self.vocab_file							):
          copyfile(self.vocab_file		,      lowerCAmelCase__							)
        elif not os.path.isfile(self.vocab_file							):
          with open(lowerCAmelCase__		,      "wb"							) as fi:
            a__	:					List[str]										=self.sp_model.serialized_model_proto()
            fi.write(lowerCAmelCase__							)

        return (out_vocab_file,)













import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import MobileNetVaImageProcessor







class 				__lowerCAmelCase       (    unittest.TestCase):







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__=7		,      lowerCAmelCase__=3		,      lowerCAmelCase__=1_8		,      lowerCAmelCase__=3_0		,      lowerCAmelCase__=4_0_0		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      )   ->						Optional[int]:



        '''simple docstring'''




        a__	:					str										=size if size is not None else {"shortest_edge": 2_0}
        a__	:					Union[str, Any]										=crop_size if crop_size is not None else {"height": 1_8, "width": 1_8}
        a__	:					Tuple										=parent
        a__	:					Optional[int]										=batch_size
        a__	:					Any										=num_channels
        a__	:					List[str]										=image_size
        a__	:					Dict										=min_resolution
        a__	:					List[Any]										=max_resolution
        a__	:					Dict										=do_resize
        a__	:					Union[str, Any]										=size
        a__	:					str										=do_center_crop
        a__	:					List[str]										=crop_size







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        return {
            "do_resize": self.do_resize,
            "size": self.size,
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
        }







@require_torch
@require_vision
class 				__lowerCAmelCase       (    UpperCamelCase__						,				unittest.TestCase):
      _lowercase				:					Optional[Any]						=			MobileNetVaImageProcessor if is_vision_available() else None







      def    _lowercase  (    self							)   ->						Tuple:



        '''simple docstring'''




        a__	:					Optional[int]										=MobileNetVaImageProcessingTester(self							)







      @property
      def    _lowercase  (    self							)   ->						List[str]:



        '''simple docstring'''




        return self.image_processor_tester.prepare_image_processor_dict()







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        a__	:					List[str]										=self.image_processing_class(**self.image_processor_dict							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "do_resize"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "size"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "do_center_crop"							)							)
        self.assertTrue(hasattr(lowerCAmelCase__		,      "crop_size"							)							)







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        a__	:					Any										=self.image_processing_class.from_dict(self.image_processor_dict							)
        self.assertEqual(image_processor.size		,      {"shortest_edge": 2_0}							)
        self.assertEqual(image_processor.crop_size		,      {"height": 1_8, "width": 1_8}							)

        a__	:					Dict										=self.image_processing_class.from_dict(self.image_processor_dict		,      size=4_2		,      crop_size=8_4							)
        self.assertEqual(image_processor.size		,      {"shortest_edge": 4_2}							)
        self.assertEqual(image_processor.crop_size		,      {"height": 8_4, "width": 8_4}							)







      def    _lowercase  (    self							)   ->						Any:



        '''simple docstring'''




        pass







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Dict										=self.image_processing_class(**self.image_processor_dict							)
        # create random PIL images
        a__	:					Optional[Any]										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      Image.Image							)

        # Test not batched input
        a__	:					List[Any]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					Dict										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )







      def    _lowercase  (    self							)   ->						int:



        '''simple docstring'''




        a__	:					str										=self.image_processing_class(**self.image_processor_dict							)
        # create random numpy tensors
        a__	:					str										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__		,      numpify=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      np.ndarray							)

        # Test not batched input
        a__	:					List[str]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					Union[str, Any]										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )







      def    _lowercase  (    self							)   ->						Optional[int]:



        '''simple docstring'''




        a__	:					Any										=self.image_processing_class(**self.image_processor_dict							)
        # create random PyTorch tensors
        a__	:					int										=prepare_image_inputs(self.image_processor_tester		,      equal_resolution=lowerCAmelCase__		,      torchify=lowerCAmelCase__							)
        for image in image_inputs:
          self.assertIsInstance(lowerCAmelCase__		,      torch.Tensor							)

        # Test not batched input
        a__	:					Optional[Any]										=image_processing(image_inputs[0]		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )

        # Test batched
        a__	:					str										=image_processing(lowerCAmelCase__		,      return_tensors="pt"							).pixel_values
        self.assertEqual(
            encoded_images.shape		,      (
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size["height"],
                self.image_processor_tester.crop_size["width"],
            )		,      )






