


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


lowerCAmelCase           =		logging.get_logger(__name__)

lowerCAmelCase           =		{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

lowerCAmelCase           =		{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

lowerCAmelCase           =		{
    '''abeja/gpt-neox-japanese-2.7b''': 2_0_4_8,
}







def        _lowerCamelCase(	lowercase__   , lowercase__	)						->	str:

		'''simple docstring'''







		with open(lowercase__   , 'r'   , encoding='utf-8'	) as f:
				__lowercase=			json.loads(f.read()	)

		__lowercase=			collections.OrderedDict()
		__lowercase=			collections.OrderedDict()
		__lowercase=			collections.OrderedDict()
		with open(lowercase__   , 'r'   , encoding='utf-8'	) as f:
				__lowercase=			f.readlines()
		__lowercase=			[[t.rstrip('\n'	)] if (t == ',' or ',' not in t) else t.rstrip('\n'	).split(','	) for t in token]
		for idx, b in enumerate(lowercase__	):
				__lowercase=			b
				__lowercase=			idx
				for wd in b:
						__lowercase=			idx

		return vocab, raw_vocab, ids_to_tokens, emoji





class       A    (   A_ ):
							UpperCamelCase_   :							Tuple       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							str       =PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							Optional[Any]       =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							str       =['''input_ids''', '''attention_mask''']


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase="<|endoftext|>"      ,				lowerCAmelCase="<|endoftext|>"      ,				lowerCAmelCase="<|startoftext|>"      ,				lowerCAmelCase="<|endoftext|>"      ,				lowerCAmelCase=False      ,				**lowerCAmelCase      ,				):
									super().__init__(
									    unk_token=lowerCAmelCase      ,				pad_token=lowerCAmelCase      ,				bos_token=lowerCAmelCase      ,				eos_token=lowerCAmelCase      ,				do_clean_text=lowerCAmelCase      ,				**lowerCAmelCase      ,				)
									if not os.path.isfile(lowerCAmelCase					):
											raise ValueError(
											    f'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
											    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
									if not os.path.isfile(lowerCAmelCase					):
											raise ValueError(
											    f'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
											    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
									__lowercase=			do_clean_text
									__lowercase,							__lowercase,							__lowercase,							__lowercase=			load_vocab_and_emoji(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			SubWordJapaneseTokenizer(
									    vocab=self.vocab      ,				ids_to_tokens=self.ids_to_tokens      ,				emoji=self.emoji					)


							@property
							def 			_A       (self					):
									# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
									return len(self.raw_vocab					)


							def 			_A       (self					):
									return dict(self.raw_vocab      ,				**self.added_tokens_encoder					)


							def 			_A       (self      ,				lowerCAmelCase					):
									return self.subword_tokenizer.tokenize(lowerCAmelCase      ,				clean=self.do_clean_text					)


							def 			_A       (self      ,				lowerCAmelCase					):
									return self.vocab.get(lowerCAmelCase      ,				self.vocab.get(self.unk_token					)					)


							def 			_A       (self      ,				lowerCAmelCase					):
									return self.subword_tokenizer.convert_id_to_token(lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			''.join(lowerCAmelCase					).strip()
									return out_string


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			[]
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(lowerCAmelCase      ,				add_special_tokens=lowerCAmelCase					) + [self.eos_token_id]					)

									if len(lowerCAmelCase					) > self.model_max_length:
											__lowercase=			input_ids[-self.model_max_length :]
									return input_ids







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									__lowercase=			0
									if os.path.isdir(lowerCAmelCase					):
											__lowercase=			os.path.join(
											    lowerCAmelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
											__lowercase=			os.path.join(
											    lowerCAmelCase      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']					)
									else:
											__lowercase=			(
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
											)
											__lowercase=			(
											    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
											)
									with open(lowerCAmelCase      ,				'w'      ,				encoding='utf-8'					) as writer:
											for token_index, token in self.ids_to_tokens.items():
													if index != token_index:
															logger.warning(
															    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
															    ' Please check that the vocabulary is not corrupted!'					)
															__lowercase=			token_index
													writer.write(','.join(lowerCAmelCase					) + '\n'					)
													index += 1
									with open(lowerCAmelCase      ,				'w'      ,				encoding='utf-8'					) as writer:
											json.dump(self.emoji      ,				lowerCAmelCase					)
									return vocab_file, emoji_file





class       A    (   A_ ):


							def __init__(self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase=			vocab  # same as swe
									__lowercase=			ids_to_tokens  # same as bpe
									__lowercase=			emoji
									__lowercase=			np.max([len(lowerCAmelCase					) for w in self.vocab.keys()]					)
									__lowercase=			re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'					)
									__lowercase=			re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'					)
									__lowercase=			re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'					)
									__lowercase=			re.compile(
									    r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
									__lowercase=			re.compile(
									    r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
									__lowercase=			re.compile(
									    r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'					)
									__lowercase=			'─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
									__lowercase=			'▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
									__lowercase=			str.maketrans({k: '<BLOCK>' for k in keisen + blocks}					)


							def __len__(self					):
									return len(self.ids_to_tokens					)


							def 			_A       (self      ,				lowerCAmelCase					):
									__lowercase=			self.content_repattera.sub('<URL>'      ,				lowerCAmelCase					)
									__lowercase=			self.content_repattera.sub('<EMAIL>'      ,				lowerCAmelCase					)
									__lowercase=			self.content_repattera.sub('<TEL>'      ,				lowerCAmelCase					)
									__lowercase=			self.content_repattera.sub('<DATE>'      ,				lowerCAmelCase					)
									__lowercase=			self.content_repattera.sub('<DATE>'      ,				lowerCAmelCase					)
									__lowercase=			self.content_repattera.sub('<PRICE>'      ,				lowerCAmelCase					)
									__lowercase=			content.translate(self.content_transa					)
									while "<BLOCK><BLOCK>" in content:
											__lowercase=			content.replace('<BLOCK><BLOCK>'      ,				'<BLOCK>'					)
									return content


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase=False					):
									__lowercase=			text.replace(' '      ,				'<SP>'					)
									__lowercase=			text.replace('　'      ,				'<SP>'					)
									__lowercase=			text.replace('\r\n'      ,				'<BR>'					)
									__lowercase=			text.replace('\n'      ,				'<BR>'					)
									__lowercase=			text.replace('\r'      ,				'<BR>'					)
									__lowercase=			text.replace('\t'      ,				'<TAB>'					)
									__lowercase=			text.replace('—'      ,				'ー'					)
									__lowercase=			text.replace('−'      ,				'ー'					)
									for k, v in self.emoji["emoji"].items():
											if k in text:
													__lowercase=			text.replace(lowerCAmelCase      ,				lowerCAmelCase					)
									if clean:
											__lowercase=			self.clean_text(lowerCAmelCase					)

									def check_simbol(lowerCAmelCase					):
											__lowercase=			x.encode()
											if len(lowerCAmelCase					) == 1 and len(lowerCAmelCase					) == 2:
													__lowercase=			(int(e[0]					) << 8) + int(e[1]					)
													if (
													    (c >= 0XC2A1 and c <= 0XC2BF)
													    or (c >= 0XC780 and c <= 0XC783)
													    or (c >= 0XCAB9 and c <= 0XCBBF)
													    or (c >= 0XCC80 and c <= 0XCDA2)
													):
															return True
											return False

									def checkuae(lowerCAmelCase					):
											__lowercase=			x.encode()
											if len(lowerCAmelCase					) == 1 and len(lowerCAmelCase					) == 3:
													__lowercase=			(int(e[0]					) << 1_6) + (int(e[1]					) << 8) + int(e[2]					)
													if c >= 0XE28080 and c <= 0XE2B07F:
															return True
											return False

									__lowercase=			0
									__lowercase=			[]
									while pos < len(lowerCAmelCase					):
											__lowercase=			min(len(lowerCAmelCase					)      ,				pos + self.maxlen + 1					) if text[pos] == '<' else pos + 3
											__lowercase=			[]  # (token_id, token, pos)
											for e in range(lowerCAmelCase      ,				lowerCAmelCase      ,				-1					):
													__lowercase=			text[pos:e]
													if wd in self.vocab:
															if wd[0] == "<" and len(lowerCAmelCase					) > 2:
																	__lowercase=			[(self.vocab[wd], wd, e)]
																	break
															else:
																	candidates.append((self.vocab[wd], wd, e)					)
											if len(lowerCAmelCase					) > 0:
													# the smallest token_id is adopted
													__lowercase,							__lowercase,							__lowercase=			sorted(lowerCAmelCase      ,				key=lambda lowerCAmelCase					: x[0]					)[0]
													result.append(lowerCAmelCase					)
													__lowercase=			e
											else:
													__lowercase=			pos + 1
													__lowercase=			text[pos:end]
													if check_simbol(lowerCAmelCase					):
															result.append('<KIGOU>'					)
													elif checkuae(lowerCAmelCase					):
															result.append('<U2000U2BFF>'					)
													else:
															for i in wd.encode('utf-8'					):
																	result.append('<|byte%d|>' % i					)
													__lowercase=			end
									return result







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase="\n"					):
									__lowercase=			[]
									__lowercase=			[]
									__lowercase=			self.ids_to_tokens[index][0]
									if word[:6] == "<|byte" and word[-2:] == "|>":
											byte_tokens.append(int(word[6:-2]					)					)
									else:
											if len(lowerCAmelCase					) > 0:
													words.append(bytearray(lowerCAmelCase					).decode('utf-8'      ,				errors='replace'					)					)
													__lowercase=			[]
											if word[:7] == "<|emoji" and word[-2:] == "|>":
													words.append(self.emoji['emoji_inv'][word]					)
											elif word == "<SP>":
													words.append(' '					)
											elif word == "<BR>":
													words.append(lowerCAmelCase					)
											elif word == "<TAB>":
													words.append('\t'					)
											elif word == "<BLOCK>":
													words.append('▀'					)
											elif word == "<KIGOU>":
													words.append('ǀ'					)
											elif word == "<U2000U2BFF>":
													words.append('‖'					)
											else:
													words.append(lowerCAmelCase					)
									if len(lowerCAmelCase					) > 0:
											words.append(bytearray(lowerCAmelCase					).decode('utf-8'      ,				errors='replace'					)					)
									__lowercase=			''.join(lowerCAmelCase					)
									return text










import collections
from typing import List, Optional, Union

from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, add_end_docstrings, add_start_docstrings, logging
from ..bert.tokenization_bert_fast import BertTokenizerFast
from .tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer, DPRReaderTokenizer


lowerCAmelCase           =		logging.get_logger(__name__)

lowerCAmelCase           =		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase           =		{
    '''vocab_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-ctx_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-ctx_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCAmelCase           =		{
    '''vocab_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-question_encoder-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-question_encoder-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCAmelCase           =		{
    '''vocab_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/vocab.txt'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/dpr-reader-single-nq-base''': (
            '''https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/tokenizer.json'''
        ),
        '''facebook/dpr-reader-multiset-base''': (
            '''https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/tokenizer.json'''
        ),
    },
}

lowerCAmelCase           =		{
    '''facebook/dpr-ctx_encoder-single-nq-base''': 5_1_2,
    '''facebook/dpr-ctx_encoder-multiset-base''': 5_1_2,
}
lowerCAmelCase           =		{
    '''facebook/dpr-question_encoder-single-nq-base''': 5_1_2,
    '''facebook/dpr-question_encoder-multiset-base''': 5_1_2,
}
lowerCAmelCase           =		{
    '''facebook/dpr-reader-single-nq-base''': 5_1_2,
    '''facebook/dpr-reader-multiset-base''': 5_1_2,
}


lowerCAmelCase           =		{
    '''facebook/dpr-ctx_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-ctx_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCAmelCase           =		{
    '''facebook/dpr-question_encoder-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-question_encoder-multiset-base''': {'''do_lower_case''': True},
}
lowerCAmelCase           =		{
    '''facebook/dpr-reader-single-nq-base''': {'''do_lower_case''': True},
    '''facebook/dpr-reader-multiset-base''': {'''do_lower_case''': True},
}





class       A    (   A_ ):
							UpperCamelCase_   :							List[Any]       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							Dict       =CONTEXT_ENCODER_PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							List[Any]       =CONTEXT_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							Optional[int]       =CONTEXT_ENCODER_PRETRAINED_INIT_CONFIGURATION
							UpperCamelCase_   :							int       =DPRContextEncoderTokenizer





class       A    (   A_ ):
							UpperCamelCase_   :							Any       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							List[str]       =QUESTION_ENCODER_PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							Optional[Any]       =QUESTION_ENCODER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							Optional[Any]       =QUESTION_ENCODER_PRETRAINED_INIT_CONFIGURATION
							UpperCamelCase_   :							List[Any]       =DPRQuestionEncoderTokenizer


lowerCAmelCase           =		collections.namedtuple(
    '''DPRSpanPrediction''', ['''span_score''', '''relevance_score''', '''doc_id''', '''start_index''', '''end_index''', '''text''']
)

lowerCAmelCase           =		collections.namedtuple('''DPRReaderOutput''', ['''start_logits''', '''end_logits''', '''relevance_logits'''])


lowerCAmelCase           =		R'''
    Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
    It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
    using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
    with the format:

    [CLS] <question token ids> [SEP] <titles ids> [SEP] <texts ids>

    Args:
        questions (`str` or `List[str]`):
            The questions to be encoded. You can specify one question for many passages. In this case, the question
            will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
            `titles` or `texts`.
        titles (`str` or `List[str]`):
            The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
        texts (`str` or `List[str]`):
            The passages texts to be encoded. This can be a string or a list of strings if there are several passages.
        padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
            Activates and controls padding. Accepts the following values:

            - `True` or `\'longest\'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
              if provided).
            - `\'max_length\'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided.
            - `False` or `\'do_not_pad\'` (default): No padding (i.e., can output a batch with sequences of different
              lengths).
        truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
            Activates and controls truncation. Accepts the following values:

            - `True` or `\'longest_first\'`: Truncate to a maximum length specified with the argument `max_length` or to
              the maximum acceptable input length for the model if that argument is not provided. This will truncate
              token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
              of pairs) is provided.
            - `\'only_first\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the first
              sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `\'only_second\'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
              acceptable input length for the model if that argument is not provided. This will only truncate the
              second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
            - `False` or `\'do_not_truncate\'` (default): No truncation (i.e., can output batch with sequence lengths
              greater than the model maximum admissible input size).
        max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
        return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `\'tf\'`: Return TensorFlow `tf.constant` objects.
                - `\'pt\'`: Return PyTorch `torch.Tensor` objects.
                - `\'np\'`: Return Numpy `np.ndarray` objects.
        return_attention_mask (`bool`, *optional*):
            Whether or not to return the attention mask. If not set, will return the attention mask according to the
            specific tokenizer\'s default, defined by the `return_outputs` attribute.

            [What are attention masks?](../glossary#attention-mask)

    Return:
        `Dict[str, List[List[int]]]`: A dictionary with the following keys:

        - `input_ids`: List of token ids to be fed to a model.
        - `attention_mask`: List of indices specifying which tokens should be attended to by the model.
    '''





@add_start_docstrings(A_ )
class       A    :


							def __call__(self      ,				lowerCAmelCase      ,				lowerCAmelCase = None      ,				lowerCAmelCase = None      ,				lowerCAmelCase = False      ,				lowerCAmelCase = False      ,				lowerCAmelCase = None      ,				lowerCAmelCase = None      ,				lowerCAmelCase = None      ,				**lowerCAmelCase      ,				):
									if titles is None and texts is None:
											return super().__call__(
											    lowerCAmelCase      ,				padding=lowerCAmelCase      ,				truncation=lowerCAmelCase      ,				max_length=lowerCAmelCase      ,				return_tensors=lowerCAmelCase      ,				return_attention_mask=lowerCAmelCase      ,				**lowerCAmelCase      ,				)
									elif titles is None or texts is None:
											__lowercase=			titles if texts is None else texts
											return super().__call__(
											    lowerCAmelCase      ,				lowerCAmelCase      ,				padding=lowerCAmelCase      ,				truncation=lowerCAmelCase      ,				max_length=lowerCAmelCase      ,				return_tensors=lowerCAmelCase      ,				return_attention_mask=lowerCAmelCase      ,				**lowerCAmelCase      ,				)
									__lowercase=			titles if not isinstance(lowerCAmelCase      ,				lowerCAmelCase					) else [titles]
									__lowercase=			texts if not isinstance(lowerCAmelCase      ,				lowerCAmelCase					) else [texts]
									__lowercase=			len(lowerCAmelCase					)
									__lowercase=			questions if not isinstance(lowerCAmelCase      ,				lowerCAmelCase					) else [questions] * n_passages
									assert len(lowerCAmelCase					) == len(
									    lowerCAmelCase					), f'There should be as many titles than texts but got {len(lowerCAmelCase					)} titles and {len(lowerCAmelCase					)} texts.'
									__lowercase=			super().__call__(lowerCAmelCase      ,				lowerCAmelCase      ,				padding=lowerCAmelCase      ,				truncation=lowerCAmelCase					)['input_ids']
									__lowercase=			super().__call__(lowerCAmelCase      ,				add_special_tokens=lowerCAmelCase      ,				padding=lowerCAmelCase      ,				truncation=lowerCAmelCase					)['input_ids']
									__lowercase=			{
									    'input_ids': [
									        (encoded_question_and_title + encoded_text)[:max_length]
									        if max_length is not None and truncation
									        else encoded_question_and_title + encoded_text
									        for encoded_question_and_title, encoded_text in zip(lowerCAmelCase      ,				lowerCAmelCase					)
									    ]
									}
									if return_attention_mask is not False:
											__lowercase=			[]
											for input_ids in encoded_inputs["input_ids"]:
													attention_mask.append([int(input_id != self.pad_token_id					) for input_id in input_ids]					)
											__lowercase=			attention_mask
									return self.pad(lowerCAmelCase      ,				padding=lowerCAmelCase      ,				max_length=lowerCAmelCase      ,				return_tensors=lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase = 1_6      ,				lowerCAmelCase = 6_4      ,				lowerCAmelCase = 4      ,				):
									__lowercase=			reader_input['input_ids']
									__lowercase,							__lowercase,							__lowercase=			reader_output[:3]
									__lowercase=			len(lowerCAmelCase					)
									__lowercase=			sorted(range(lowerCAmelCase					)      ,				reverse=lowerCAmelCase      ,				key=relevance_logits.__getitem__					)
									__lowercase=			[]
									for doc_id in sorted_docs:
											__lowercase=			list(input_ids[doc_id]					)
											# assuming question & title information is at the beginning of the sequence
											__lowercase=			sequence_ids.index(self.sep_token_id      ,				2					) + 1  # second sep id
											if sequence_ids[-1] == self.pad_token_id:
													__lowercase=			sequence_ids.index(self.pad_token_id					)
											else:
													__lowercase=			len(lowerCAmelCase					)

											__lowercase=			self._get_best_spans(
											    start_logits=start_logits[doc_id][passage_offset:sequence_len]      ,				end_logits=end_logits[doc_id][passage_offset:sequence_len]      ,				max_answer_length=lowerCAmelCase      ,				top_spans=lowerCAmelCase      ,				)
											for start_index, end_index in best_spans:
													start_index += passage_offset
													end_index += passage_offset
													nbest_spans_predictions.append(
													    DPRSpanPrediction(
													        span_score=start_logits[doc_id][start_index] + end_logits[doc_id][end_index]      ,				relevance_score=relevance_logits[doc_id]      ,				doc_id=lowerCAmelCase      ,				start_index=lowerCAmelCase      ,				end_index=lowerCAmelCase      ,				text=self.decode(sequence_ids[start_index : end_index + 1]					)      ,				)					)
											if len(lowerCAmelCase					) >= num_spans:
													break
									return nbest_spans_predictions[:num_spans]







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				):
									__lowercase=			[]
									for start_index, start_score in enumerate(lowerCAmelCase					):
											for answer_length, end_score in enumerate(end_logits[start_index : start_index + max_answer_length]					):
													scores.append(((start_index, start_index + answer_length), start_score + end_score)					)
									__lowercase=			sorted(lowerCAmelCase      ,				key=lambda lowerCAmelCase					: x[1]      ,				reverse=lowerCAmelCase					)
									__lowercase=			[]
									for (start_index, end_index), score in scores:
											assert start_index <= end_index, f'Wrong span indices: [{start_index}:{end_index}]'
											__lowercase=			end_index - start_index + 1
											assert length <= max_answer_length, f'Span is too long: {length} > {max_answer_length}'
											if any(
											    start_index <= prev_start_index <= prev_end_index <= end_index
											    or prev_start_index <= start_index <= end_index <= prev_end_index
											    for (prev_start_index, prev_end_index) in chosen_span_intervals					):
													continue
											chosen_span_intervals.append((start_index, end_index)					)

											if len(lowerCAmelCase					) == top_spans:
													break
									return chosen_span_intervals





@add_end_docstrings(A_ )
class       A    (   A_					,				A_ ):
							UpperCamelCase_   :							Optional[int]       =VOCAB_FILES_NAMES
							UpperCamelCase_   :							List[str]       =READER_PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_   :							Dict       =READER_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_   :							Optional[Any]       =READER_PRETRAINED_INIT_CONFIGURATION
							UpperCamelCase_   :							Union[str, Any]       =['''input_ids''', '''attention_mask''']
							UpperCamelCase_   :							Dict       =DPRReaderTokenizer






