


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase__    :Dict       =				logging.get_logger(__name__)

lowerCAmelCase__    :List[str]       =				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase__    :str       =				{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase__    :Union[str, Any]       =				{
    '''google/realm-cc-news-pretrained-embedder''': 5_1_2,
    '''google/realm-cc-news-pretrained-encoder''': 5_1_2,
    '''google/realm-cc-news-pretrained-scorer''': 5_1_2,
    '''google/realm-cc-news-pretrained-openqa''': 5_1_2,
    '''google/realm-orqa-nq-openqa''': 5_1_2,
    '''google/realm-orqa-nq-reader''': 5_1_2,
    '''google/realm-orqa-wq-openqa''': 5_1_2,
    '''google/realm-orqa-wq-reader''': 5_1_2,
}

lowerCAmelCase__    :List[Any]       =				{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}

class 						__a (							UpperCAmelCase     ):
		_a      :    int           =      VOCAB_FILES_NAMES
		_a      :    List[Any]           =      PRETRAINED_VOCAB_FILES_MAP
		_a      :    List[str]           =      PRETRAINED_INIT_CONFIGURATION
		_a      :    str           =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a      :    List[Any]           =      RealmTokenizer
		def __init__(     self     ,       _SCREAMING_SNAKE_CASE=None     ,       _SCREAMING_SNAKE_CASE=None     ,       _SCREAMING_SNAKE_CASE=True     ,       _SCREAMING_SNAKE_CASE="[UNK]"     ,       _SCREAMING_SNAKE_CASE="[SEP]"     ,       _SCREAMING_SNAKE_CASE="[PAD]"     ,       _SCREAMING_SNAKE_CASE="[CLS]"     ,       _SCREAMING_SNAKE_CASE="[MASK]"     ,       _SCREAMING_SNAKE_CASE=True     ,       _SCREAMING_SNAKE_CASE=None     ,       **_SCREAMING_SNAKE_CASE     ,       )					->			str:




									"""simple docstring"""





									super().__init__(
									    _SCREAMING_SNAKE_CASE     ,       tokenizer_file=_SCREAMING_SNAKE_CASE     ,       do_lower_case=_SCREAMING_SNAKE_CASE     ,       unk_token=_SCREAMING_SNAKE_CASE     ,       sep_token=_SCREAMING_SNAKE_CASE     ,       pad_token=_SCREAMING_SNAKE_CASE     ,       cls_token=_SCREAMING_SNAKE_CASE     ,       mask_token=_SCREAMING_SNAKE_CASE     ,       tokenize_chinese_chars=_SCREAMING_SNAKE_CASE     ,       strip_accents=_SCREAMING_SNAKE_CASE     ,       **_SCREAMING_SNAKE_CASE     ,       )

									_UpperCAmelCase											=     json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
									if (
									    normalizer_state.get('lowercase'     ,       _SCREAMING_SNAKE_CASE		) != do_lower_case
									    or normalizer_state.get('strip_accents'     ,       _SCREAMING_SNAKE_CASE		) != strip_accents
									    or normalizer_state.get('handle_chinese_chars'     ,       _SCREAMING_SNAKE_CASE		) != tokenize_chinese_chars
									):
																_UpperCAmelCase											=     getattr(_SCREAMING_SNAKE_CASE     ,       normalizer_state.pop('type'		)		)
																_UpperCAmelCase											=     do_lower_case
																_UpperCAmelCase											=     strip_accents
																_UpperCAmelCase											=     tokenize_chinese_chars
																_UpperCAmelCase											=     normalizer_class(**_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     do_lower_case
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       **_SCREAMING_SNAKE_CASE		)					->			str:




									"""simple docstring"""





									_UpperCAmelCase											=     PaddingStrategy.MAX_LENGTH

									_UpperCAmelCase											=     text
									_UpperCAmelCase											=     kwargs.pop('text_pair'     ,       _SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase											=     kwargs.pop('return_tensors'     ,       _SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     {
									    'input_ids': [],
									    'attention_mask': [],
									    'token_type_ids': [],
									}

									for idx, candidate_text in enumerate(_SCREAMING_SNAKE_CASE		):
																if batch_text_pair is not None:
																							_UpperCAmelCase											=     batch_text_pair[idx]
																else:
																							_UpperCAmelCase											=     None

																_UpperCAmelCase											=     super().__call__(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors=_SCREAMING_SNAKE_CASE     ,       **_SCREAMING_SNAKE_CASE		)

																_UpperCAmelCase											=     encoded_candidates.get('input_ids'		)
																_UpperCAmelCase											=     encoded_candidates.get('attention_mask'		)
																_UpperCAmelCase											=     encoded_candidates.get('token_type_ids'		)

																if encoded_input_ids is not None:
																							output_data["input_ids"].append(_SCREAMING_SNAKE_CASE		)
																if encoded_attention_mask is not None:
																							output_data["attention_mask"].append(_SCREAMING_SNAKE_CASE		)
																if encoded_token_type_ids is not None:
																							output_data["token_type_ids"].append(_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase											=     {key: item for key, item in output_data.items() if len(_SCREAMING_SNAKE_CASE		) != 0}

									return BatchEncoding(_SCREAMING_SNAKE_CASE     ,       tensor_type=_SCREAMING_SNAKE_CASE		)
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE=None		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

									return output
		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE = None		)					->			List[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     [self.sep_token_id]
									_UpperCAmelCase											=     [self.cls_token_id]
									if token_ids_a is None:
																return len(cls + token_ids_a + sep		) * [0]
									return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






		def       UpperCAmelCase__							(     self     ,       _SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE = None		)					->			Tuple[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     self._tokenizer.model.save(_SCREAMING_SNAKE_CASE     ,       name=_SCREAMING_SNAKE_CASE		)
									return tuple(_SCREAMING_SNAKE_CASE		)




from __future__ import annotations

def 			lowerCAmelCase__					(     a__:						str     ,    a__:						str			)    ->						bool:


							'''simple docstring'''





							_UpperCAmelCase											=     get_failure_array(a__			)

							# 2) Step through text searching for pattern
							_UpperCAmelCase						,			_UpperCAmelCase											=     0, 0  # index into text, pattern
							while i < len(a__			):
														if pattern[j] == text[i]:
																					if j == (len(a__			) - 1):
																												return True
																					j += 1

														# if this is a prefix in our pattern
														# just go back far enough to continue
														elif j > 0:
																					_UpperCAmelCase											=     failure[j - 1]
																					continue
														i += 1
							return False

def 			lowerCAmelCase__					(     a__:						str			)    ->						list[int]:


							'''simple docstring'''





							_UpperCAmelCase											=     [0]
							_UpperCAmelCase											=     0
							_UpperCAmelCase											=     1
							while j < len(a__			):
														if pattern[i] == pattern[j]:
																					i += 1
														elif i > 0:
																					_UpperCAmelCase											=     failure[i - 1]
																					continue
														j += 1
														failure.append(a__			)
							return failure


if __name__ == "__main__":
		# Test 1)
		lowerCAmelCase__    :List[Any]       =				'''abc1abc12'''
		lowerCAmelCase__    :Optional[int]       =				'''alskfjaldsabc1abc1abc12k23adsfabcabc'''
		lowerCAmelCase__    :int       =				'''alskfjaldsk23adsfabcabc'''
		assert kmp(pattern, texta) and not kmp(pattern, texta)

		# Test 2)
		lowerCAmelCase__    :List[str]       =				'''ABABX'''
		lowerCAmelCase__    :int       =				'''ABABZABABYABABX'''
		assert kmp(pattern, text)

		# Test 3)
		lowerCAmelCase__    :int       =				'''AAAB'''
		lowerCAmelCase__    :Any       =				'''ABAAAAAB'''
		assert kmp(pattern, text)

		# Test 4)
		lowerCAmelCase__    :Union[str, Any]       =				'''abcdabcy'''
		lowerCAmelCase__    :Any       =				'''abcxabcdabxabcdabcdabcy'''
		assert kmp(pattern, text)

		# Test 5)
		lowerCAmelCase__    :List[Any]       =				'''aabaabaaa'''
		assert get_failure_array(pattern) == [0, 1, 0, 1, 2, 3, 4, 5, 2]
