





"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


a						: Tuple     	=			logging.get_logger(__name__)

a						: Dict     	=			{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a						: Tuple     	=			{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

a						: Optional[int]     	=			{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

a						: str     	=			{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}


class        __UpperCamelCase				(						a__						):
							lowerCamelCase   :					str      					=VOCAB_FILES_NAMES
							lowerCamelCase   :					Tuple      					=PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase   :					List[str]      					=PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase   :					List[Any]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase   :					Optional[int]      					=RealmTokenizer






							def __init__(   self      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=True      ,					lowerCAmelCase__="[UNK]"      ,					lowerCAmelCase__="[SEP]"      ,					lowerCAmelCase__="[PAD]"      ,					lowerCAmelCase__="[CLS]"      ,					lowerCAmelCase__="[MASK]"      ,					lowerCAmelCase__=True      ,					lowerCAmelCase__=None      ,					**lowerCAmelCase__      ,					)	-> Union[str, Any]:
								super().__init__(
								    lowerCAmelCase__      ,					tokenizer_file=lowerCAmelCase__      ,					do_lower_case=lowerCAmelCase__      ,					unk_token=lowerCAmelCase__      ,					sep_token=lowerCAmelCase__      ,					pad_token=lowerCAmelCase__      ,					cls_token=lowerCAmelCase__      ,					mask_token=lowerCAmelCase__      ,					tokenize_chinese_chars=lowerCAmelCase__      ,					strip_accents=lowerCAmelCase__      ,					**lowerCAmelCase__      ,					)

								a      :		Union[str, Any]	=				json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
								if (
								    normalizer_state.get("lowercase"      ,					lowerCAmelCase__      ) != do_lower_case
								    or normalizer_state.get("strip_accents"      ,					lowerCAmelCase__      ) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"      ,					lowerCAmelCase__      ) != tokenize_chinese_chars
								):
									a      :		Any	=				getattr(lowerCAmelCase__      ,					normalizer_state.pop("type"      )      )
									a      :		int	=				do_lower_case
									a      :		str	=				strip_accents
									a      :		str	=				tokenize_chinese_chars
									a      :		Dict	=				normalizer_class(**lowerCAmelCase__      )

								a      :		Tuple	=				do_lower_case






							def     __a   (   self      ,					lowerCAmelCase__      ,					**lowerCAmelCase__      )	-> List[Any]:
								a      :		List[str]	=				PaddingStrategy.MAX_LENGTH

								a      :		str	=				text
								a      :		Optional[int]	=				kwargs.pop("text_pair"      ,					lowerCAmelCase__      )
								a      :		List[Any]	=				kwargs.pop("return_tensors"      ,					lowerCAmelCase__      )

								a      :		Optional[Any]	=				{
								    "input_ids": [],
								    "attention_mask": [],
								    "token_type_ids": [],
								}

								for idx, candidate_text in enumerate(lowerCAmelCase__      ):
									if batch_text_pair is not None:
										a      :		int	=				batch_text_pair[idx]
									else:
										a      :		Tuple	=				None

									a      :		Tuple	=				super().__call__(lowerCAmelCase__      ,					lowerCAmelCase__      ,					return_tensors=lowerCAmelCase__      ,					**lowerCAmelCase__      )

									a      :		Any	=				encoded_candidates.get("input_ids"      )
									a      :		Union[str, Any]	=				encoded_candidates.get("attention_mask"      )
									a      :		Optional[int]	=				encoded_candidates.get("token_type_ids"      )

									if encoded_input_ids is not None:
										output_data["input_ids"].append(lowerCAmelCase__      )
									if encoded_attention_mask is not None:
										output_data["attention_mask"].append(lowerCAmelCase__      )
									if encoded_token_type_ids is not None:
										output_data["token_type_ids"].append(lowerCAmelCase__      )

								a      :		Optional[Any]	=				{key: item for key, item in output_data.items() if len(lowerCAmelCase__      ) != 0}

								return BatchEncoding(lowerCAmelCase__      ,					tensor_type=lowerCAmelCase__      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      )	-> Any:
								a      :		List[str]	=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Optional[int]	=				[self.sep_token_id]
								a      :		Optional[int]	=				[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep      ) * [0]
								return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> Tuple[str]:
								a      :		Optional[Any]	=				self._tokenizer.model.save(lowerCAmelCase__      ,					name=lowerCAmelCase__      )
								return tuple(lowerCAmelCase__      )









"""simple docstring"""




from __future__ import annotations

import math

import numpy as np
from numpy.linalg import norm






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->float:






	'''simple docstring'''






	return math.sqrt(sum(pow(a - b    ,       2					) for a, b in zip(_lowercase    ,       _lowercase					)					)					)






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->list[list[list[float] | float]]:






	'''simple docstring'''






	if dataset.ndim != value_array.ndim:
		a      :		str	=				(
		    "Wrong input data's dimensions... "
		    F"""dataset : {dataset.ndim}, value_array : {value_array.ndim}"""
		)
		raise ValueError(_lowercase					)

	try:
		if dataset.shape[1] != value_array.shape[1]:
			a      :		int	=				(
			    "Wrong input data's shape... "
			    F"""dataset : {dataset.shape[1]}, value_array : {value_array.shape[1]}"""
			)
			raise ValueError(_lowercase					)
	except IndexError:
		if dataset.ndim != value_array.ndim:
			raise TypeError("Wrong shape"					)

	if dataset.dtype != value_array.dtype:
		a      :		Union[str, Any]	=				(
		    "Input data have different datatype... "
		    F"""dataset : {dataset.dtype}, value_array : {value_array.dtype}"""
		)
		raise TypeError(_lowercase					)

	a      :		str	=				[]

	for value in value_array:
		a      :		List[Any]	=				euclidean(_lowercase    ,       dataset[0]					)
		a      :		str	=				dataset[0].tolist()

		for dataset_value in dataset[1:]:
			a      :		Tuple	=				euclidean(_lowercase    ,       _lowercase					)

			if dist > temp_dist:
				a      :		Dict	=				temp_dist
				a      :		Optional[int]	=				dataset_value.tolist()

		answer.append([vector, dist]					)

	return answer






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    np.ndarray    ,       _lowercase						:    np.ndarray					) ->float:






	'''simple docstring'''






	return np.dot(_lowercase    ,       _lowercase					) / (norm(_lowercase					) * norm(_lowercase					))


if __name__ == "__main__":
		import doctest

		doctest.testmod()


