"""simple docstring"""




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase     :				List[str]    				=    logging.get_logger(__name__)

__lowerCAmelCase     :				List[Any]    				=    "▁"

__lowerCAmelCase     :				List[Any]    				=    {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__lowerCAmelCase     :				int    				=    {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__lowerCAmelCase     :				List[str]    				=    {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__lowerCAmelCase     :				Union[str, Any]    				=    {
    "ernie-m-base": 5_14,
    "ernie-m-large": 5_14,
}

__lowerCAmelCase     :				str    				=    {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}

class    a_   (  __UpperCamelCase      ):
					UpperCamelCase_			:						List[str]      =     ["input_ids"]

					UpperCamelCase_			:						Any      =     VOCAB_FILES_NAMES
					UpperCamelCase_			:						Dict      =     PRETRAINED_INIT_CONFIGURATION
					UpperCamelCase_			:						List[Any]      =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCamelCase_			:						Dict      =     PRETRAINED_VOCAB_FILES_MAP
					UpperCamelCase_			:						str      =     RESOURCE_FILES_NAMES




					def __init__(						self    :  List[str]	,    snake_case__    :  Optional[Any]	,    snake_case__    :  Union[str, Any]=None	,    snake_case__    :  List[Any]=False	,    snake_case__    :  Optional[Any]="utf8"	,    snake_case__    :  List[str]="[UNK]"	,    snake_case__    :  Any="[SEP]"	,    snake_case__    :  Optional[Any]="[PAD]"	,    snake_case__    :  Optional[int]="[CLS]"	,    snake_case__    :  Dict="[MASK]"	,    snake_case__    :  Optional[Dict[str, Any]] = None	,    **snake_case__    :  Optional[Any]	,    ):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.

							lowerCAmelCase__   = {} if sp_model_kwargs is None else sp_model_kwargs
							super().__init__(
							    do_lower_case=snake_case__	,    unk_token=snake_case__	,    sep_token=snake_case__	,    pad_token=snake_case__	,    cls_token=snake_case__	,    mask_token=snake_case__	,    vocab_file=snake_case__	,    encoding=snake_case__	,    sp_model_kwargs=self.sp_model_kwargs	,    **snake_case__	,    )
							lowerCAmelCase__   = do_lower_case
							lowerCAmelCase__   = sentencepiece_model_ckpt
							lowerCAmelCase__   = spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(snake_case__  )

							# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
							if vocab_file is not None:
									lowerCAmelCase__   = self.load_vocab(filepath=snake_case__  )
							else:
									lowerCAmelCase__   = {self.sp_model.id_to_piece(snake_case__  ): id for id in range(self.sp_model.get_piece_size()  )}
							lowerCAmelCase__   = {v: k for k, v in self.vocab.items()}




					def     _SCREAMING_SNAKE_CASE    (						self    :  int	,    snake_case__    :  int  ):
							if text is None:
									return None

							lowerCAmelCase__   = self.tokenize(snake_case__  )
							lowerCAmelCase__ ,       lowerCAmelCase__   = """""", []

							for i, ch in enumerate(snake_case__  ):
									if ch in self.SP_CHAR_MAPPING:
											lowerCAmelCase__   = self.SP_CHAR_MAPPING.get(snake_case__  )
									else:
											lowerCAmelCase__   = unicodedata.normalize("""NFKC"""	,    snake_case__  )
									if self.is_whitespace(snake_case__  ):
											continue
									normalized_text += ch
									char_mapping.extend([i] * len(snake_case__  )  )

							lowerCAmelCase__ ,       lowerCAmelCase__ ,       lowerCAmelCase__   = normalized_text, [], 0

							if self.do_lower_case:
									lowerCAmelCase__   = text.lower()

							for token in split_tokens:
									if token[:1] == "▁":
											lowerCAmelCase__   = token[1:]
									lowerCAmelCase__   = text[offset:].index(snake_case__  ) + offset
									lowerCAmelCase__   = start + len(snake_case__  )

									token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)  )
									lowerCAmelCase__   = end
							return token_mapping




					@property
					def     _SCREAMING_SNAKE_CASE    (						self    :  Union[str, Any]  ):
							return len(self.vocab  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Union[str, Any]  ):
							return dict(self.vocab	,    **self.added_tokens_encoder  )




					def __getstate__(						self    :  int  ):
							lowerCAmelCase__   = self.__dict__.copy()
							lowerCAmelCase__   = None
							return state




					def __setstate__(						self    :  Union[str, Any]	,    snake_case__    :  List[Any]  ):
							lowerCAmelCase__   = d

							# for backward compatibility
							if not hasattr(self	,    """sp_model_kwargs"""  ):
									lowerCAmelCase__   = {}

							lowerCAmelCase__   = spm.SentencePieceProcessor(**self.sp_model_kwargs  )
							self.sp_model.Load(self.sentencepiece_model_ckpt  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  str	,    snake_case__    :  List[Any]  ):
							return "".join((self.SP_CHAR_MAPPING.get(snake_case__	,    snake_case__  ) for c in text)  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Dict	,    snake_case__    :  int	,    snake_case__    :  List[Any]=False	,    snake_case__    :  int=64	,    snake_case__    :  Optional[int]=0.1  ):

							if self.sp_model_kwargs.get("""enable_sampling"""  ) is True:
									lowerCAmelCase__   = True
							if self.sp_model_kwargs.get("""alpha"""  ) is not None:
									lowerCAmelCase__   = self.sp_model_kwargs.get("""alpha"""  )
							if self.sp_model_kwargs.get("""nbest_size"""  ) is not None:
									lowerCAmelCase__   = self.sp_model_kwargs.get("""nbest_size"""  )

							if not enable_sampling:
									lowerCAmelCase__   = self.sp_model.EncodeAsPieces(snake_case__  )
							else:
									lowerCAmelCase__   = self.sp_model.SampleEncodeAsPieces(snake_case__	,    snake_case__	,    snake_case__  )
							lowerCAmelCase__   = []
							for pi, piece in enumerate(snake_case__  ):
									if piece == SPIECE_UNDERLINE:
											if not pieces[pi + 1].startswith(snake_case__  ) and pi != 0:
													new_pieces.append(snake_case__  )
													continue
											else:
													continue
									lowerCAmelCase__   = 0
									for i, chunk in enumerate(snake_case__  ):
											if chunk == SPIECE_UNDERLINE:
													continue
											if self.is_ch_char(snake_case__  ) or self.is_punct(snake_case__  ):
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
															new_pieces.append(piece[lst_i:i]  )
													new_pieces.append(snake_case__  )
													lowerCAmelCase__   = i + 1
											elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
															new_pieces.append(piece[lst_i:i]  )
													lowerCAmelCase__   = i
											elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
															new_pieces.append(piece[lst_i:i]  )
													lowerCAmelCase__   = i
									if len(snake_case__  ) > lst_i:
											new_pieces.append(piece[lst_i:]  )
							return new_pieces




					def     _SCREAMING_SNAKE_CASE    (						self    :  str	,    snake_case__    :  Optional[Any]  ):
							lowerCAmelCase__   = """""".join(snake_case__  ).replace(snake_case__	,    """ """  ).strip()
							return out_string




					def     _SCREAMING_SNAKE_CASE    (						self    :  int	,    snake_case__    :  int  ):
							lowerCAmelCase__   = self.convert_ids_to_tokens(snake_case__  )
							lowerCAmelCase__   = """""".join(snake_case__  ).replace(snake_case__	,    """ """  ).strip()
							return out_string




					def     _SCREAMING_SNAKE_CASE    (						self    :  Dict	,    snake_case__    :  Dict  ):
							return self.vocab.get(snake_case__	,    self.vocab.get(self.unk_token  )  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  List[Any]	,    snake_case__    :  Dict  ):
							return self.reverse_vocab.get(snake_case__	,    self.unk_token  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Dict	,    snake_case__    :  str	,    snake_case__    :  Optional[int]=None  ):
							if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							lowerCAmelCase__   = [self.cls_token_id]
							lowerCAmelCase__   = [self.sep_token_id]
							return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




					def     _SCREAMING_SNAKE_CASE    (						self    :  Any	,    snake_case__    :  Optional[int]	,    snake_case__    :  str=None  ):
							if offset_mapping_a is None:
									return [(0, 0)] + offset_mapping_a + [(0, 0)]

							return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




					def     _SCREAMING_SNAKE_CASE    (						self    :  List[str]	,    snake_case__    :  Optional[Any]	,    snake_case__    :  List[str]=None	,    snake_case__    :  List[Any]=False  ):

							if already_has_special_tokens:
									if token_ids_a is not None:
											raise ValueError(
											    """You should not supply a second sequence if the provided sequence of """
											    """ids is already formatted with special tokens for the model."""  )
									return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is not None:
									return [1] + ([0] * len(snake_case__  )) + [1, 1] + ([0] * len(snake_case__  )) + [1]
							return [1] + ([0] * len(snake_case__  )) + [1]




					def     _SCREAMING_SNAKE_CASE    (						self    :  List[Any]	,    snake_case__    :  List[int]	,    snake_case__    :  Optional[List[int]] = None  ):
							# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
							if token_ids_a is None:
									# [CLS] X [SEP]
									return (len(snake_case__  ) + 2) * [0]

							# [CLS] A [SEP] [SEP] B [SEP]
							return [0] * (len(snake_case__  ) + 1) + [1] * (len(snake_case__  ) + 3)




					def     _SCREAMING_SNAKE_CASE    (						self    :  Tuple	,    snake_case__    :  List[Any]  ):
							if "\u4e00" <= char <= "\u9fff":
									return True
							return False




					def     _SCREAMING_SNAKE_CASE    (						self    :  Optional[int]	,    snake_case__    :  Optional[int]  ):
							if ("a" <= char <= "z") or ("A" <= char <= "Z"):
									return True
							return False




					def     _SCREAMING_SNAKE_CASE    (						self    :  str	,    snake_case__    :  Union[str, Any]  ):
							if char in ",;:.?!~，；：。？！《》【】":
									return True
							return False




					def     _SCREAMING_SNAKE_CASE    (						self    :  List[str]	,    snake_case__    :  str  ):
							if char == " " or char == "\t" or char == "\n" or char == "\r":
									return True
							if len(snake_case__  ) == 1:
									lowerCAmelCase__   = unicodedata.category(snake_case__  )
									if cat == "Zs":
											return True
							return False




					def     _SCREAMING_SNAKE_CASE    (						self    :  Any	,    snake_case__    :  str  ):
							lowerCAmelCase__   = {}
							with io.open(snake_case__	,    """r"""	,    encoding="""utf-8"""  ) as f:
									for index, line in enumerate(snake_case__  ):
											lowerCAmelCase__   = line.rstrip("""\n"""  )
											lowerCAmelCase__   = int(snake_case__  )

							return token_to_idx




					def     _SCREAMING_SNAKE_CASE    (						self    :  Any	,    snake_case__    :  str	,    snake_case__    :  Optional[str] = None  ):
							lowerCAmelCase__   = 0
							if os.path.isdir(snake_case__  ):
									lowerCAmelCase__   = os.path.join(
									    snake_case__	,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
							else:
									lowerCAmelCase__   = (filename_prefix + """-""" if filename_prefix else """""") + save_directory
							with open(snake_case__	,    """w"""	,    encoding="""utf-8"""  ) as writer:
									for token, token_index in sorted(self.vocab.items()	,    key=lambda snake_case__  : kv[1]  ):
											if index != token_index:
													logger.warning(
													    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
													    """ Please check that the vocabulary is not corrupted!"""  )
													lowerCAmelCase__   = token_index
											writer.write(token + """\n"""  )
											index += 1

							lowerCAmelCase__   = os.path.join(snake_case__	,    """sentencepiece.bpe.model"""  )
							with open(snake_case__	,    """wb"""  ) as fi:
									lowerCAmelCase__   = self.sp_model.serialized_model_proto()
									fi.write(snake_case__  )

							return (vocab_file,)





"""simple docstring"""




from typing import Optional, Tuple

import jax
import jax.numpy as jnp
from flax import linen as nn
from flax.core.frozen_dict import FrozenDict
from transformers import CLIPConfig, FlaxPreTrainedModel
from transformers.models.clip.modeling_flax_clip import FlaxCLIPVisionModule







def 				_UpperCAmelCase		(							lowerCamelCase__      ,	lowerCamelCase__      ,	lowerCamelCase__=1e-12  ):





		"""simple docstring"""




		lowerCAmelCase__   = jnp.divide(emb_a.T      ,	jnp.clip(jnp.linalg.norm(lowerCamelCase__      ,	axis=1  )      ,	a_min=lowerCamelCase__  )  ).T
		lowerCAmelCase__   = jnp.divide(emb_a.T      ,	jnp.clip(jnp.linalg.norm(lowerCamelCase__      ,	axis=1  )      ,	a_min=lowerCamelCase__  )  ).T
		return jnp.matmul(lowerCamelCase__      ,	norm_emb_a.T  )

class    a_   (  nn.Module      ):
					UpperCamelCase_			:						CLIPConfig
					UpperCamelCase_			:						jnp.dtype      =     jnp.floataa




					def     _SCREAMING_SNAKE_CASE    (						self    :  Tuple  ):
							lowerCAmelCase__   = FlaxCLIPVisionModule(self.config.vision_config  )
							lowerCAmelCase__   = nn.Dense(self.config.projection_dim	,    use_bias=snake_case__	,    dtype=self.dtype  )

							lowerCAmelCase__   = self.param("""concept_embeds"""	,    jax.nn.initializers.ones	,    (17, self.config.projection_dim)  )
							lowerCAmelCase__   = self.param(
							    """special_care_embeds"""	,    jax.nn.initializers.ones	,    (3, self.config.projection_dim)  )

							lowerCAmelCase__   = self.param("""concept_embeds_weights"""	,    jax.nn.initializers.ones	,    (17,)  )
							lowerCAmelCase__   = self.param("""special_care_embeds_weights"""	,    jax.nn.initializers.ones	,    (3,)  )




					def __call__(						self    :  Optional[int]	,    snake_case__    :  int  ):
							lowerCAmelCase__   = self.vision_model(snake_case__  )[1]
							lowerCAmelCase__   = self.visual_projection(snake_case__  )

							lowerCAmelCase__   = jax_cosine_distance(snake_case__	,    self.special_care_embeds  )
							lowerCAmelCase__   = jax_cosine_distance(snake_case__	,    self.concept_embeds  )

							# increase this value to create a stronger `nfsw` filter
							# at the cost of increasing the possibility of filtering benign image inputs
							lowerCAmelCase__   = 0.0

							lowerCAmelCase__   = special_cos_dist - self.special_care_embeds_weights[None, :] + adjustment
							lowerCAmelCase__   = jnp.round(snake_case__	,    3  )
							lowerCAmelCase__   = jnp.any(special_scores > 0	,    axis=1	,    keepdims=snake_case__  )
							# Use a lower threshold if an image has any special care concept
							lowerCAmelCase__   = is_special_care * 0.01

							lowerCAmelCase__   = cos_dist - self.concept_embeds_weights[None, :] + special_adjustment
							lowerCAmelCase__   = jnp.round(snake_case__	,    3  )
							lowerCAmelCase__   = jnp.any(concept_scores > 0	,    axis=1  )

							return has_nsfw_concepts

class    a_   (  __UpperCamelCase      ):
					UpperCamelCase_			:						Any      =     CLIPConfig
					UpperCamelCase_			:						str      =     "clip_input"
					UpperCamelCase_			:						str      =     FlaxStableDiffusionSafetyCheckerModule




					def __init__(						self    :  Optional[int]	,    snake_case__    :  CLIPConfig	,    snake_case__    :  Optional[Tuple] = None	,    snake_case__    :  int = 0	,    snake_case__    :  jnp.dtype = jnp.floataa	,    snake_case__    :  bool = True	,    **snake_case__    :  str	,    ):
							if input_shape is None:
									lowerCAmelCase__   = (1, 224, 224, 3)
							lowerCAmelCase__   = self.module_class(config=snake_case__	,    dtype=snake_case__	,    **snake_case__  )
							super().__init__(snake_case__	,    snake_case__	,    input_shape=snake_case__	,    seed=snake_case__	,    dtype=snake_case__	,    _do_init=_do_init  )




					def     _SCREAMING_SNAKE_CASE    (						self    :  Tuple	,    snake_case__    :  jax.random.KeyArray	,    snake_case__    :  Tuple	,    snake_case__    :  FrozenDict = None  ):
							# init input tensor
							lowerCAmelCase__   = jax.random.normal(snake_case__	,    snake_case__  )

							lowerCAmelCase__ ,       lowerCAmelCase__   = jax.random.split(snake_case__  )
							lowerCAmelCase__   = {"""params""": params_rng, """dropout""": dropout_rng}

							lowerCAmelCase__   = self.module.init(snake_case__	,    snake_case__  )["""params"""]

							return random_params




					def __call__(						self    :  Union[str, Any]	,    snake_case__    :  Union[str, Any]	,    snake_case__    :  dict = None	,    ):
							lowerCAmelCase__   = jnp.transpose(snake_case__	,    (0, 2, 3, 1)  )

							return self.module.apply(
							    {"""params""": params or self.params}	,    jnp.array(snake_case__	,    dtype=jnp.floataa  )	,    rngs={}	,    )




