




import copy

from ...configuration_utils import PretrainedConfig
from ...utils import add_start_docstrings


_A     =			R'''
    [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and
    can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.

    Args:
        title_sep (`str`, *optional*, defaults to  `" / "`):
            Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].
        doc_sep (`str`, *optional*, defaults to  `" // "`):
            Separator inserted between the text of the retrieved document and the original input when calling
            [`RagRetriever`].
        n_docs (`int`, *optional*, defaults to 5):
            Number of documents to retrieve.
        max_combined_length (`int`, *optional*, defaults to 300):
            Max length of contextualized input returned by [`~RagRetriever.__call__`].
        retrieval_vector_size (`int`, *optional*, defaults to 768):
            Dimensionality of the document embeddings indexed by [`RagRetriever`].
        retrieval_batch_size (`int`, *optional*, defaults to 8):
            Retrieval batch size, defined as the number of queries issues concurrently to the faiss index encapsulated
            [`RagRetriever`].
        dataset (`str`, *optional*, defaults to `"wiki_dpr"`):
            A dataset identifier of the indexed dataset in HuggingFace Datasets (list all available datasets and ids
            using `datasets.list_datasets()`).
        dataset_split (`str`, *optional*, defaults to `"train"`)
            Which split of the `dataset` to load.
        index_name (`str`, *optional*, defaults to `"compressed"`)
            The index name of the index associated with the `dataset`. One can choose between `"legacy"`, `"exact"` and
            `"compressed"`.
        index_path (`str`, *optional*)
            The path to the serialized faiss index on disk.
        passages_path (`str`, *optional*):
            A path to text passages compatible with the faiss index. Required if using
            [`~models.rag.retrieval_rag.LegacyIndex`]
        use_dummy_dataset (`bool`, *optional*, defaults to `False`)
            Whether to load a "dummy" variant of the dataset specified by `dataset`.
        label_smoothing (`float`, *optional*, defaults to 0.0):
            Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing
            in the loss calculation. If set to 0, no label smoothing is performed.
        do_marginalize (`bool`, *optional*, defaults to `False`):
            If `True`, the logits are marginalized over all documents by making use of
            `torch.nn.functional.log_softmax`.
        reduce_loss (`bool`, *optional*, defaults to `False`):
            Whether or not to reduce the NLL loss using the `torch.Tensor.sum` operation.
        do_deduplication (`bool`, *optional*, defaults to `True`):
            Whether or not to deduplicate the generations from different context documents for a given input. Has to be
            set to `False` if used while training with distributed backend.
        exclude_bos_score (`bool`, *optional*, defaults to `False`):
            Whether or not to disregard the BOS token when computing the loss.
        output_retrieved(`bool`, *optional*, defaults to `False`):
            If set to `True`, `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and
            `context_attention_mask` are returned. See returned tensors for more detail.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models).
        forced_eos_token_id (`int`, *optional*):
            The id of the token to force as the last generated token when `max_length` is reached. Usually set to
            `eos_token_id`.
'''


@add_start_docstrings(__UpperCAmelCase     )
class  A		(  __UpperCAmelCase     ):
		__snake_case									=   'rag'
		__snake_case									=   True



		def __init__(     self,	UpperCamelCase__=None,	UpperCamelCase__=True,	UpperCamelCase__=None,	UpperCamelCase__=None,	UpperCamelCase__=None,	UpperCamelCase__=None,	UpperCamelCase__=None,	UpperCamelCase__=" / ",	UpperCamelCase__=" // ",	UpperCamelCase__=5,	UpperCamelCase__=300,	UpperCamelCase__=768,	UpperCamelCase__=8,	UpperCamelCase__="wiki_dpr",	UpperCamelCase__="train",	UpperCamelCase__="compressed",	UpperCamelCase__=None,	UpperCamelCase__=None,	UpperCamelCase__=False,	UpperCamelCase__=False,	UpperCamelCase__=0.0,	UpperCamelCase__=True,	UpperCamelCase__=False,	UpperCamelCase__=False,	UpperCamelCase__=False,	UpperCamelCase__=True,	UpperCamelCase__=None,	**UpperCamelCase__,	):







								"""simple docstring"""



								super().__init__(
								    bos_token_id=UpperCamelCase__,	pad_token_id=UpperCamelCase__,	eos_token_id=UpperCamelCase__,	decoder_start_token_id=UpperCamelCase__,	forced_eos_token_id=UpperCamelCase__,	is_encoder_decoder=UpperCamelCase__,	prefix=UpperCamelCase__,	vocab_size=UpperCamelCase__,	**UpperCamelCase__,	)
								assert (
								    "question_encoder" in kwargs and "generator" in kwargs
								), "Config has to be initialized with question_encoder and generator config"
								lowerCAmelCase_      			=  kwargs.pop('''question_encoder'''   )
								lowerCAmelCase_      			=  question_encoder_config.pop('''model_type'''   )
								lowerCAmelCase_      			=  kwargs.pop('''generator'''   )
								lowerCAmelCase_      			=  decoder_config.pop('''model_type'''   )

								from ..auto.configuration_auto import AutoConfig

								lowerCAmelCase_      			=  AutoConfig.for_model(UpperCamelCase__,	**UpperCamelCase__   )
								lowerCAmelCase_      			=  AutoConfig.for_model(UpperCamelCase__,	**UpperCamelCase__   )

								lowerCAmelCase_      			=  reduce_loss
								lowerCAmelCase_      			=  label_smoothing
								lowerCAmelCase_      			=  exclude_bos_score
								lowerCAmelCase_      			=  do_marginalize

								lowerCAmelCase_      			=  title_sep
								lowerCAmelCase_      			=  doc_sep
								lowerCAmelCase_      			=  n_docs
								lowerCAmelCase_      			=  max_combined_length

								lowerCAmelCase_      			=  dataset
								lowerCAmelCase_      			=  dataset_split
								lowerCAmelCase_      			=  index_name

								lowerCAmelCase_      			=  retrieval_vector_size
								lowerCAmelCase_      			=  retrieval_batch_size
								lowerCAmelCase_      			=  passages_path
								lowerCAmelCase_      			=  index_path
								lowerCAmelCase_      			=  use_dummy_dataset

								lowerCAmelCase_      			=  output_retrieved

								lowerCAmelCase_      			=  do_deduplication

								lowerCAmelCase_      			=  use_cache

								if self.forced_eos_token_id is None:
														lowerCAmelCase_      			=  getattr(self.generator,	'''forced_eos_token_id''',	UpperCamelCase__   )



		@classmethod
		def        SCREAMING_SNAKE_CASE__			(     cls,	UpperCamelCase__,	UpperCamelCase__,	**UpperCamelCase__   ):







								"""simple docstring"""



								return cls(question_encoder=question_encoder_config.to_dict(),	generator=generator_config.to_dict(),	**UpperCamelCase__   )







		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  copy.deepcopy(self.__dict__   )
								lowerCAmelCase_      			=  self.question_encoder.to_dict()
								lowerCAmelCase_      			=  self.generator.to_dict()
								lowerCAmelCase_      			=  self.__class__.model_type
								return output










import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def __UpperCamelCase			(       _A ,  _A  ):
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''' ,  [False, True]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A ,  keep_in_memory=_A  ).read()
						_check_json_dataset(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						_check_json_dataset(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





def __UpperCamelCase			(       _A ,  _A  ):
						# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
						lowerCAmelCase_      			=  {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
						lowerCAmelCase_      			=  features.copy()
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''split''' ,  [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A ,  split=_A  ).read()
						_check_json_dataset(_A ,  _A  )
						assert dataset.split == split if split else "train"





@pytest.mark.parametrize('''path_type''' ,  [str, list]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						if issubclass(_A ,  _A  ):
												lowerCAmelCase_      			=  jsonl_path
						elif issubclass(_A ,  _A  ):
												lowerCAmelCase_      			=  [jsonl_path]
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A  ).read()
						_check_json_dataset(_A ,  _A  )





def __UpperCamelCase			(       _A ,  _A ,  _A=("train",)  ):
						assert isinstance(_A ,  _A  )
						for split in splits:
												lowerCAmelCase_      			=  dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''' ,  [False, True]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowerCAmelCase_      			=  JsonDatasetReader({'''train''': jsonl_path} ,  cache_dir=_A ,  keep_in_memory=_A  ).read()
						_check_json_datasetdict(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader({'''train''': jsonl_path} ,  features=_A ,  cache_dir=_A  ).read()
						_check_json_datasetdict(_A ,  _A  )





@pytest.mark.parametrize('''split''' ,  [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						if split:
												lowerCAmelCase_      			=  {split: jsonl_path}
						else:
												lowerCAmelCase_      			=  '''train'''
												lowerCAmelCase_      			=  {'''train''': jsonl_path, '''test''': jsonl_path}
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A  ).read()
						_check_json_datasetdict(_A ,  _A ,  splits=list(path.keys()  )  )
						assert all(dataset[split].split == split for split in path.keys()  )





def __UpperCamelCase			(       _A  ):
						return json.load(_A  )





def __UpperCamelCase			(       _A  ):
						return [json.loads(_A  ) for line in buffer]






class  A		:



		@pytest.mark.parametrize('''lines, load_json_function''',	[(True, load_json_lines), (False, load_json)]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json_function(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								assert isinstance(exported_content[0],	UpperCamelCase__   )
								assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at''',	[
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''   ), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ],	)
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	orient=UpperCamelCase__   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(UpperCamelCase__,	'''keys'''   ) and not hasattr(exported_content[0],	'''keys'''   )
								if len_at:
														assert len(exported_content[len_at]   ) == 10
								else:
														assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize('''lines, load_json_function''',	[(True, load_json_lines), (False, load_json)]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	num_proc=2   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json_function(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								assert isinstance(exported_content[0],	UpperCamelCase__   )
								assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at''',	[
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''   ), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ],	)
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	orient=UpperCamelCase__,	num_proc=2   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(UpperCamelCase__,	'''keys'''   ) and not hasattr(exported_content[0],	'''keys'''   )
								if len_at:
														assert len(exported_content[len_at]   ) == 10
								else:
														assert len(UpperCamelCase__   ) == 10



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								with pytest.raises(UpperCamelCase__   ):
														with io.BytesIO() as buffer:
																				JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	num_proc=0   )







		@pytest.mark.parametrize('''compression, extension''',	[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  tmp_path_factory.mktemp('''data'''   ) / f"test.json.{extension}"
								lowerCAmelCase_      			=  str(shared_datadir / f"test_file.json.{extension}"   )
								JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	compression=UpperCamelCase__   ).write()

								with fsspec.open(UpperCamelCase__,	'''rb''',	compression='''infer'''   ) as f:
														lowerCAmelCase_      			=  f.read()
								with fsspec.open(UpperCamelCase__,	'''rb''',	compression='''infer'''   ) as f:
														lowerCAmelCase_      			=  f.read()
								assert exported_content == original_content




