





"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a		:					Any	      =     logging.get_logger(__name__)
a		:					int	      =     {'''vocab_file''': '''spiece.model'''}

a		:					int	      =     {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

a		:					Union[str, Any]	      =     {
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

a		:					Optional[int]	      =     '''▁'''


class 			__UpperCamelCase							(		a__		):
							lowerCamelCase  :   List[Any]    =VOCAB_FILES_NAMES
							lowerCamelCase  :   int    =PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase  :   Dict    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






							def __init__(  self ,   lowerCAmelCase__ ,   lowerCAmelCase__=True ,   lowerCAmelCase__=True ,   lowerCAmelCase__=False ,   lowerCAmelCase__="[CLS]" ,   lowerCAmelCase__="[SEP]" ,   lowerCAmelCase__="<unk>" ,   lowerCAmelCase__="[SEP]" ,   lowerCAmelCase__="<pad>" ,   lowerCAmelCase__="[CLS]" ,   lowerCAmelCase__="[MASK]" ,   lowerCAmelCase__ = None ,   **lowerCAmelCase__ ,   )      ->					None:
								# Mask token behave like a normal word, i.e. include the space before it and
								# is included in the raw text, there should be a match in a non-normalized sentence.
								a :						str      =							(
								    AddedToken(lowerCAmelCase__ ,   lstrip=lowerCAmelCase__ ,   rstrip=lowerCAmelCase__ ,   normalized=lowerCAmelCase__ )
								    if isinstance(lowerCAmelCase__ ,   lowerCAmelCase__ )
								    else mask_token
								)

								a :						Any      =							{} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=lowerCAmelCase__ ,   remove_space=lowerCAmelCase__ ,   keep_accents=lowerCAmelCase__ ,   bos_token=lowerCAmelCase__ ,   eos_token=lowerCAmelCase__ ,   unk_token=lowerCAmelCase__ ,   sep_token=lowerCAmelCase__ ,   pad_token=lowerCAmelCase__ ,   cls_token=lowerCAmelCase__ ,   mask_token=lowerCAmelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCAmelCase__ ,   )

								a :						Dict      =							do_lower_case
								a :						Any      =							remove_space
								a :						Any      =							keep_accents
								a :						Any      =							vocab_file

								a :						Any      =							spm.SentencePieceProcessor(**self.sp_model_kwargs )
								self.sp_model.Load(lowerCAmelCase__ )






							@property
							def        __a  (  self )      ->					str:
								return len(self.sp_model )






							def        __a  (  self )      ->					Optional[int]:
								a :						Tuple      =							{self.convert_ids_to_tokens(lowerCAmelCase__ ): i for i in range(self.vocab_size )}
								vocab.update(self.added_tokens_encoder )
								return vocab






							def __getstate__(  self )      ->					Union[str, Any]:
								a :						Dict      =							self.__dict__.copy()
								a :						List[Any]      =							None
								return state






							def __setstate__(  self ,   lowerCAmelCase__ )      ->					List[Any]:
								a :						List[Any]      =							d

								# for backward compatibility
								if not hasattr(self ,   "sp_model_kwargs" ):
									a :						Any      =							{}

								a :						Optional[Any]      =							spm.SentencePieceProcessor(**self.sp_model_kwargs )
								self.sp_model.Load(self.vocab_file )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					str:
								if self.remove_space:
									a :						Any      =							" ".join(inputs.strip().split() )
								else:
									a :						int      =							inputs
								a :						str      =							outputs.replace("``" ,   "\"" ).replace("''" ,   "\"" )

								if not self.keep_accents:
									a :						Dict      =							unicodedata.normalize("NFKD" ,   lowerCAmelCase__ )
									a :						Optional[int]      =							"".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__ )] )
								if self.do_lower_case:
									a :						Any      =							outputs.lower()

								return outputs






							def        __a  (  self ,   lowerCAmelCase__ )      ->					List[str]:
								a :						Dict      =							self.preprocess_text(lowerCAmelCase__ )
								a :						Dict      =							self.sp_model.encode(lowerCAmelCase__ ,   out_type=lowerCAmelCase__ )
								a :						List[str]      =							[]
								for piece in pieces:
									if len(lowerCAmelCase__ ) > 1 and piece[-1] == str("," ) and piece[-2].isdigit():
										a :						List[Any]      =							self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__ ,   "" ) )
										if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
											if len(cur_pieces[0] ) == 1:
												a :						Tuple      =							cur_pieces[1:]
											else:
												a :						int      =							cur_pieces[0][1:]
										cur_pieces.append(piece[-1] )
										new_pieces.extend(lowerCAmelCase__ )
									else:
										new_pieces.append(lowerCAmelCase__ )

								return new_pieces






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Any:
								return self.sp_model.PieceToId(lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Dict:
								return self.sp_model.IdToPiece(lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Dict:
								a :						Optional[int]      =							[]
								a :						Tuple      =							""
								a :						Optional[Any]      =							False
								for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
										if not prev_is_special:
											out_string += " "
										out_string += self.sp_model.decode(lowerCAmelCase__ ) + token
										a :						Optional[int]      =							True
										a :						int      =							[]
									else:
										current_sub_tokens.append(lowerCAmelCase__ )
										a :						str      =							False
								out_string += self.sp_model.decode(lowerCAmelCase__ )
								return out_string.strip()






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					List[int]:
								a :						Optional[int]      =							[self.sep_token_id]
								a :						Dict      =							[self.cls_token_id]
								if token_ids_a is None:
									return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None ,   lowerCAmelCase__ = False )      ->					List[int]:

								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase__ ,   token_ids_a=lowerCAmelCase__ ,   already_has_special_tokens=lowerCAmelCase__ )

								if token_ids_a is not None:
									return [1] + ([0] * len(lowerCAmelCase__ )) + [1] + ([0] * len(lowerCAmelCase__ )) + [1]
								return [1] + ([0] * len(lowerCAmelCase__ )) + [1]






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					List[int]:
								a :						Optional[Any]      =							[self.sep_token_id]
								a :						Optional[int]      =							[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep ) * [0]
								return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






							def        __a  (  self ,   lowerCAmelCase__ ,   lowerCAmelCase__ = None )      ->					Tuple[str]:
								if not os.path.isdir(lowerCAmelCase__ ):
									logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
									return
								a :						List[Any]      =							os.path.join(
								    lowerCAmelCase__ ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

								if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase__ ) and os.path.isfile(self.vocab_file ):
									copyfile(self.vocab_file ,   lowerCAmelCase__ )
								elif not os.path.isfile(self.vocab_file ):
									with open(lowerCAmelCase__ ,   "wb" ) as fi:
										a :						List[str]      =							self.sp_model.serialized_model_proto()
										fi.write(lowerCAmelCase__ )

								return (out_vocab_file,)









"""simple docstring"""




import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
		import os

		import jax.numpy as jnp
		from jax import jit

		from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
		from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

		a		:					Optional[Any]	      =     '''0.12'''  # assumed parallelism: 8


if is_torch_available():
		import torch






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       List[str]       ,			_lowercase		:       Any       ,			_lowercase		:       Union[str, Any]=None   )					->Union[str, Any]:






	'''simple docstring'''






	if rng is None:
		a :						Dict      =							random.Random()

	a :						Any      =							1
	for dim in shape:
		total_dims *= dim

	a :						Dict      =							[]
	for _ in range(_lowercase   ):
		values.append(rng.randint(0       ,			vocab_size - 1   )   )

	a :						Dict      =							np.array(_lowercase       ,			dtype=jnp.intaa   ).reshape(_lowercase   )

	return output






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       List[str]       ,			_lowercase		:       int=None   )					->Optional[Any]:






	'''simple docstring'''






	a :						str      =							ids_tensor(_lowercase       ,			vocab_size=2       ,			rng=_lowercase   )
	# make sure that at least one token is attended to for each batch
	a :						Union[str, Any]      =							1
	return attn_mask


@require_flax
class 			__UpperCamelCase							:
							lowerCamelCase  :   Dict    =None
							lowerCamelCase  :   List[Any]    =()






							def        __a  (  self )      ->					Union[str, Any]:
								a,			a :						List[Any]      =							self.model_tester.prepare_config_and_inputs_for_common()

								# cut to half length & take max batch_size 3
								a :						Any      =							2
								a :						List[Any]      =							inputs["input_ids"].shape[-1] // 2
								a :						Union[str, Any]      =							inputs["input_ids"][:max_batch_size, :sequence_length]

								a :						Optional[Any]      =							jnp.ones_like(lowerCAmelCase__ )
								a :						Any      =							attention_mask[:max_batch_size, :sequence_length]

								# generate max 5 tokens
								a :						int      =							input_ids.shape[-1] + 5
								if config.eos_token_id is not None and config.pad_token_id is None:
									# hack to allow generate for models such as GPT2 as is done in `generate()`
									a :						List[str]      =							config.eos_token_id
								return config, input_ids, attention_mask, max_length






							@is_pt_flax_cross_test
							def        __a  (  self )      ->					List[Any]:
								a,			a,			a,			a :						int      =							self._get_input_ids_and_config()
								a :						Dict      =							False
								a :						Tuple      =							max_length
								a :						List[Any]      =							0

								for model_class in self.all_generative_model_classes:
									a :						List[str]      =							model_class(lowerCAmelCase__ )

									a :						int      =							model_class.__name__[4:]  # Skip the "Flax" at the beginning
									a :						List[Any]      =							getattr(lowerCAmelCase__ ,   lowerCAmelCase__ )
									a :						Any      =							pt_model_class(lowerCAmelCase__ ).eval()
									a :						int      =							load_flax_weights_in_pytorch_model(lowerCAmelCase__ ,   flax_model.params )

									a :						Optional[Any]      =							flax_model.generate(lowerCAmelCase__ ).sequences
									a :						Tuple      =							pt_model.generate(torch.tensor(lowerCAmelCase__ ,   dtype=torch.long ) )

									if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
										a :						List[str]      =							flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

									self.assertListEqual(pt_generation_outputs.numpy().tolist() ,   flax_generation_outputs.tolist() )






							def        __a  (  self )      ->					Optional[Any]:
								a,			a,			a,			a :						Optional[Any]      =							self._get_input_ids_and_config()
								a :						List[str]      =							False
								a :						Tuple      =							max_length

								for model_class in self.all_generative_model_classes:
									a :						Dict      =							model_class(lowerCAmelCase__ )

									a :						Union[str, Any]      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						Dict      =							jit(model.generate )
									a :						int      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					Any:
								a,			a,			a,			a :						List[Any]      =							self._get_input_ids_and_config()
								a :						Dict      =							True
								a :						Any      =							max_length

								for model_class in self.all_generative_model_classes:
									a :						int      =							model_class(lowerCAmelCase__ )

									a :						Optional[int]      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						List[Any]      =							jit(model.generate )
									a :						List[Any]      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					Dict:
								a,			a,			a,			a :						str      =							self._get_input_ids_and_config()
								a :						str      =							False
								a :						Union[str, Any]      =							max_length
								a :						Optional[int]      =							2

								for model_class in self.all_generative_model_classes:
									a :						List[Any]      =							model_class(lowerCAmelCase__ )

									a :						Optional[Any]      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						Union[str, Any]      =							jit(model.generate )
									a :						Optional[int]      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					Optional[Any]:
								a,			a,			a,			a :						Optional[Any]      =							self._get_input_ids_and_config()
								a :						Optional[Any]      =							False
								a :						Tuple      =							max_length
								a :						List[Any]      =							2
								a :						Any      =							2

								for model_class in self.all_generative_model_classes:
									a :						Any      =							model_class(lowerCAmelCase__ )

									a :						List[str]      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[0] ,   input_ids.shape[0] * config.num_return_sequences )






							def        __a  (  self )      ->					Optional[int]:
								a,			a,			a,			a :						List[str]      =							self._get_input_ids_and_config()
								a :						Optional[int]      =							True
								a :						Dict      =							max_length
								a :						Dict      =							0.8
								a :						Any      =							10
								a :						Union[str, Any]      =							0.3
								a :						List[Any]      =							1
								a :						Optional[int]      =							8
								a :						List[Any]      =							9

								for model_class in self.all_generative_model_classes:
									a :						Tuple      =							model_class(lowerCAmelCase__ )

									a :						Any      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						Tuple      =							jit(model.generate )
									a :						Dict      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					Optional[int]:
								a,			a,			a,			a :						Dict      =							self._get_input_ids_and_config()
								a :						Dict      =							max_length
								a :						List[str]      =							1
								a :						List[str]      =							8
								a :						List[str]      =							9

								for model_class in self.all_generative_model_classes:
									a :						Dict      =							model_class(lowerCAmelCase__ )

									a :						str      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						List[str]      =							jit(model.generate )
									a :						Union[str, Any]      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					int:
								a,			a,			a,			a :						Tuple      =							self._get_input_ids_and_config()
								a :						List[Any]      =							max_length
								a :						Dict      =							2
								a :						List[Any]      =							1
								a :						List[str]      =							8
								a :						Any      =							9

								for model_class in self.all_generative_model_classes:
									a :						Optional[int]      =							model_class(lowerCAmelCase__ )

									a :						List[str]      =							model.generate(lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						int      =							jit(model.generate )
									a :						Optional[int]      =							jit_generate(lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					Optional[int]:
								a,			a,			a,			a :						List[str]      =							self._get_input_ids_and_config()

								# pad attention mask on the left
								a :						str      =							attention_mask.at[(0, 0)].set(0 )

								a :						Any      =							False
								a :						Union[str, Any]      =							max_length

								for model_class in self.all_generative_model_classes:
									a :						Union[str, Any]      =							model_class(lowerCAmelCase__ )

									a :						Union[str, Any]      =							model.generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						List[Any]      =							jit(model.generate )
									a :						int      =							jit_generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					List[str]:
								a,			a,			a,			a :						Union[str, Any]      =							self._get_input_ids_and_config()

								# pad attention mask on the left
								a :						Any      =							attention_mask.at[(0, 0)].set(0 )

								a :						Dict      =							True
								a :						List[Any]      =							max_length

								for model_class in self.all_generative_model_classes:
									a :						List[str]      =							model_class(lowerCAmelCase__ )

									a :						int      =							model.generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						Any      =							jit(model.generate )
									a :						List[str]      =							jit_generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )






							def        __a  (  self )      ->					str:
								a,			a,			a,			a :						Dict      =							self._get_input_ids_and_config()

								# pad attention mask on the left
								a :						Union[str, Any]      =							attention_mask.at[(0, 0)].set(0 )

								a :						int      =							2
								a :						Any      =							max_length

								for model_class in self.all_generative_model_classes:
									a :						Optional[Any]      =							model_class(lowerCAmelCase__ )

									a :						List[Any]      =							model.generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences
									self.assertEqual(generation_outputs.shape[-1] ,   lowerCAmelCase__ )

									a :						List[Any]      =							jit(model.generate )
									a :						Dict      =							jit_generate(lowerCAmelCase__ ,   attention_mask=lowerCAmelCase__ ).sequences

									self.assertListEqual(generation_outputs.tolist() ,   jit_generation_outputs.tolist() )


@require_flax
class 			__UpperCamelCase							(		unittest.TestCase		):






							def        __a  (  self )      ->					List[Any]:
								a :						Any      =							AutoTokenizer.from_pretrained("hf-internal-testing/tiny-bert" )
								a :						Tuple      =							FlaxAutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-bert-flax-only" )

								a :						int      =							"Hello world"
								a :						int      =							tokenizer(lowerCAmelCase__ ,   return_tensors="np" ).input_ids

								# typos are quickly detected (the correct argument is `do_sample`)
								with self.assertRaisesRegex(lowerCAmelCase__ ,   "do_samples" ):
									model.generate(lowerCAmelCase__ ,   do_samples=lowerCAmelCase__ )

								# arbitrary arguments that will not be used anywhere are also not accepted
								with self.assertRaisesRegex(lowerCAmelCase__ ,   "foo" ):
									a :						str      =							{"foo": "bar"}
									model.generate(lowerCAmelCase__ ,   **lowerCAmelCase__ )


