

import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY






@is_pipeline_test
@require_torch_or_tf
class        A_		(       unittest.TestCase				):
							_UpperCAmelCase    :  str						      =      MODEL_FOR_CAUSAL_LM_MAPPING
							_UpperCAmelCase    :  int						      =      TF_MODEL_FOR_CAUSAL_LM_MAPPING
							@require_torch
							def     lowerCAmelCase     (		self		:			List[str]):
								__lowerCamelCase				:				Optional[Any]					       = pipeline(task='text-generation'   ,model='sshleifer/tiny-ctrl'   ,framework='pt')
								# Using `do_sample=False` to force deterministic output
								__lowerCamelCase				:				List[Any]					       = text_generator('This is a test'   ,do_sample=UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {
								            'generated_text': (
								                'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
								                ' oscope. FiliFili@@'
								            )
								        }
								    ]   ,)

								__lowerCamelCase				:				int					       = text_generator(['This is a test', 'This is a second test'])
								self.assertEqual(
								    UpperCamelCase__   ,[
								        [
								            {
								                'generated_text': (
								                    'This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope.'
								                    ' oscope. FiliFili@@'
								                )
								            }
								        ],
								        [
								            {
								                'generated_text': (
								                    'This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy'
								                    ' oscope. oscope. FiliFili@@'
								                )
								            }
								        ],
								    ]   ,)

								__lowerCamelCase				:				int					       = text_generator('This is a test'   ,do_sample=UpperCamelCase__   ,num_return_sequences=2   ,return_tensors=UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {'generated_token_ids': ANY(UpperCamelCase__)},
								        {'generated_token_ids': ANY(UpperCamelCase__)},
								    ]   ,)
								__lowerCamelCase				:				Tuple					       = text_generator.model.config.eos_token_id
								__lowerCamelCase				:				Any					       = '<pad>'
								__lowerCamelCase				:				List[str]					       = text_generator(
								    ['This is a test', 'This is a second test']   ,do_sample=UpperCamelCase__   ,num_return_sequences=2   ,batch_size=2   ,return_tensors=UpperCamelCase__   ,)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        [
								            {'generated_token_ids': ANY(UpperCamelCase__)},
								            {'generated_token_ids': ANY(UpperCamelCase__)},
								        ],
								        [
								            {'generated_token_ids': ANY(UpperCamelCase__)},
								            {'generated_token_ids': ANY(UpperCamelCase__)},
								        ],
								    ]   ,)
							@require_tf
							def     lowerCAmelCase     (		self		:			Tuple):
								__lowerCamelCase				:				Dict					       = pipeline(task='text-generation'   ,model='sshleifer/tiny-ctrl'   ,framework='tf')

								# Using `do_sample=False` to force deterministic output
								__lowerCamelCase				:				Optional[int]					       = text_generator('This is a test'   ,do_sample=UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {
								            'generated_text': (
								                'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
								                ' please,'
								            )
								        }
								    ]   ,)

								__lowerCamelCase				:				Union[str, Any]					       = text_generator(['This is a test', 'This is a second test']   ,do_sample=UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        [
								            {
								                'generated_text': (
								                    'This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵'
								                    ' please,'
								                )
								            }
								        ],
								        [
								            {
								                'generated_text': (
								                    'This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes'
								                    ' Cannes 閲閲Cannes Cannes Cannes 攵 please,'
								                )
								            }
								        ],
								    ]   ,)
							def     lowerCAmelCase     (		self		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			List[str]):
								__lowerCamelCase				:				Optional[Any]					       = TextGenerationPipeline(model=UpperCamelCase__   ,tokenizer=UpperCamelCase__)
								return text_generator, ["This is a test", "Another test"]
							def     lowerCAmelCase     (		self		:			str):
								__lowerCamelCase				:				Tuple					       = 'Hello I believe in'
								__lowerCamelCase				:				int					       = pipeline('text-generation'   ,model='hf-internal-testing/tiny-random-gpt2')
								__lowerCamelCase				:				List[Any]					       = text_generator(UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[{'generated_text': 'Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe'}]   ,)

								__lowerCamelCase				:				Optional[int]					       = text_generator(UpperCamelCase__   ,stop_sequence=' fe')
								self.assertEqual(UpperCamelCase__   ,[{'generated_text': 'Hello I believe in fe'}])
							def     lowerCAmelCase     (		self		:			Optional[Any]   ,SCREAMING_SNAKE_CASE__		:			int   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]):
								__lowerCamelCase				:				Optional[int]					       = text_generator.model
								__lowerCamelCase				:				List[str]					       = text_generator.tokenizer

								__lowerCamelCase				:				int					       = text_generator('This is a test')
								self.assertEqual(UpperCamelCase__   ,[{'generated_text': ANY(UpperCamelCase__)}])
								self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'))

								__lowerCamelCase				:				Optional[Any]					       = text_generator('This is a test'   ,return_full_text=UpperCamelCase__)
								self.assertEqual(UpperCamelCase__   ,[{'generated_text': ANY(UpperCamelCase__)}])
								self.assertNotIn('This is a test'   ,outputs[0]['generated_text'])

								__lowerCamelCase				:				Union[str, Any]					       = pipeline(task='text-generation'   ,model=UpperCamelCase__   ,tokenizer=UpperCamelCase__   ,return_full_text=UpperCamelCase__)
								__lowerCamelCase				:				List[Any]					       = text_generator('This is a test')
								self.assertEqual(UpperCamelCase__   ,[{'generated_text': ANY(UpperCamelCase__)}])
								self.assertNotIn('This is a test'   ,outputs[0]['generated_text'])

								__lowerCamelCase				:				str					       = text_generator('This is a test'   ,return_full_text=UpperCamelCase__)
								self.assertEqual(UpperCamelCase__   ,[{'generated_text': ANY(UpperCamelCase__)}])
								self.assertTrue(outputs[0]['generated_text'].startswith('This is a test'))

								__lowerCamelCase				:				Optional[int]					       = text_generator(['This is great !', 'Something else']   ,num_return_sequences=2   ,do_sample=UpperCamelCase__)
								self.assertEqual(
								    UpperCamelCase__   ,[
								        [{'generated_text': ANY(UpperCamelCase__)}, {'generated_text': ANY(UpperCamelCase__)}],
								        [{'generated_text': ANY(UpperCamelCase__)}, {'generated_text': ANY(UpperCamelCase__)}],
								    ]   ,)

								if text_generator.tokenizer.pad_token is not None:
									__lowerCamelCase				:				List[Any]					       = text_generator(
									    ['This is great !', 'Something else']   ,num_return_sequences=2   ,batch_size=2   ,do_sample=UpperCamelCase__)
									self.assertEqual(
									    UpperCamelCase__   ,[
									        [{'generated_text': ANY(UpperCamelCase__)}, {'generated_text': ANY(UpperCamelCase__)}],
									        [{'generated_text': ANY(UpperCamelCase__)}, {'generated_text': ANY(UpperCamelCase__)}],
									    ]   ,)

								with self.assertRaises(UpperCamelCase__):
									__lowerCamelCase				:				Any					       = text_generator('test'   ,return_full_text=UpperCamelCase__   ,return_text=UpperCamelCase__)
								with self.assertRaises(UpperCamelCase__):
									__lowerCamelCase				:				int					       = text_generator('test'   ,return_full_text=UpperCamelCase__   ,return_tensors=UpperCamelCase__)
								with self.assertRaises(UpperCamelCase__):
									__lowerCamelCase				:				Optional[Any]					       = text_generator('test'   ,return_text=UpperCamelCase__   ,return_tensors=UpperCamelCase__)

								# Empty prompt is slighly special
								# it requires BOS token to exist.
								# Special case for Pegasus which will always append EOS so will
								# work even without BOS.
								if (
								    text_generator.tokenizer.bos_token_id is not None
								    or "Pegasus" in tokenizer.__class__.__name__
								    or "Git" in model.__class__.__name__
								):
									__lowerCamelCase				:				str					       = text_generator('')
									self.assertEqual(UpperCamelCase__   ,[{'generated_text': ANY(UpperCamelCase__)}])
								else:
									with self.assertRaises((ValueError, AssertionError)):
										__lowerCamelCase				:				Optional[int]					       = text_generator('')

								if text_generator.framework == "tf":
									# TF generation does not support max_new_tokens, and it's impossible
									# to control long generation with only max_length without
									# fancy calculation, dismissing tests for now.
									return
								# We don't care about infinite range models.
								# They already work.
								# Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
								__lowerCamelCase				:				int					       = ['RwkvForCausalLM', 'XGLMForCausalLM', 'GPTNeoXForCausalLM']
								if (
								    tokenizer.model_max_length < 1_0_0_0_0
								    and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
								):
									# Handling of large generations
									with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)):
										text_generator('This is a test' * 5_0_0   ,max_new_tokens=2_0)

									__lowerCamelCase				:				int					       = text_generator('This is a test' * 5_0_0   ,handle_long_generation='hole'   ,max_new_tokens=2_0)
									# Hole strategy cannot work
									with self.assertRaises(UpperCamelCase__):
										text_generator(
										    'This is a test' * 5_0_0   ,handle_long_generation='hole'   ,max_new_tokens=tokenizer.model_max_length + 1_0   ,)
							@require_torch
							@require_accelerate
							@require_torch_gpu
							def     lowerCAmelCase     (		self		:			Any):
								import torch

								# Classic `model_kwargs`
								__lowerCamelCase				:				Dict					       = pipeline(
								    model='hf-internal-testing/tiny-random-bloom'   ,model_kwargs={'device_map': 'auto', 'torch_dtype': torch.bfloataa}   ,)
								self.assertEqual(pipe.model.device   ,torch.device(0))
								self.assertEqual(pipe.model.lm_head.weight.dtype   ,torch.bfloataa)
								__lowerCamelCase				:				int					       = pipe('This is a test')
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {
								            'generated_text': (
								                'This is a test test test test test test test test test test test test test test test test'
								                ' test'
								            )
								        }
								    ]   ,)

								# Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
								__lowerCamelCase				:				List[str]					       = pipeline(model='hf-internal-testing/tiny-random-bloom'   ,device_map='auto'   ,torch_dtype=torch.bfloataa)
								self.assertEqual(pipe.model.device   ,torch.device(0))
								self.assertEqual(pipe.model.lm_head.weight.dtype   ,torch.bfloataa)
								__lowerCamelCase				:				Any					       = pipe('This is a test')
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {
								            'generated_text': (
								                'This is a test test test test test test test test test test test test test test test test'
								                ' test'
								            )
								        }
								    ]   ,)

								# torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
								__lowerCamelCase				:				Union[str, Any]					       = pipeline(model='hf-internal-testing/tiny-random-bloom'   ,device_map='auto')
								self.assertEqual(pipe.model.device   ,torch.device(0))
								self.assertEqual(pipe.model.lm_head.weight.dtype   ,torch.floataa)
								__lowerCamelCase				:				Tuple					       = pipe('This is a test')
								self.assertEqual(
								    UpperCamelCase__   ,[
								        {
								            'generated_text': (
								                'This is a test test test test test test test test test test test test test test test test'
								                ' test'
								            )
								        }
								    ]   ,)
							@require_torch
							@require_torch_gpu
							def     lowerCAmelCase     (		self		:			Any):
								import torch

								__lowerCamelCase				:				Any					       = pipeline(model='hf-internal-testing/tiny-random-bloom'   ,device=0   ,torch_dtype=torch.floataa)
								pipe('This is a test')
							@require_torch
							@require_accelerate
							@require_torch_gpu
							def     lowerCAmelCase     (		self		:			int):
								import torch

								__lowerCamelCase				:				List[Any]					       = pipeline(model='hf-internal-testing/tiny-random-bloom'   ,device_map='auto'   ,torch_dtype=torch.floataa)
								pipe('This is a test'   ,do_sample=UpperCamelCase__   ,top_p=0.5)


							def     lowerCAmelCase     (		self		:			Tuple):
								__lowerCamelCase				:				Union[str, Any]					       = 'Hello world'
								__lowerCamelCase				:				List[Any]					       = pipeline('text-generation'   ,model='hf-internal-testing/tiny-random-gpt2')
								if text_generator.model.framework == "tf":
									__lowerCamelCase				:				Optional[int]					       = logging.get_logger('transformers.generation.tf_utils')
								else:
									__lowerCamelCase				:				Optional[Any]					       = logging.get_logger('transformers.generation.utils')
								__lowerCamelCase				:				List[Any]					       = 'Both `max_new_tokens`'  # The beggining of the message to be checked in this test

								# Both are set by the user -> log warning
								with CaptureLogger(UpperCamelCase__) as cl:
									__lowerCamelCase				:				Dict					       = text_generator(UpperCamelCase__   ,max_length=1_0   ,max_new_tokens=1)
								self.assertIn(UpperCamelCase__   ,cl.out)

								# The user only sets one -> no warning
								with CaptureLogger(UpperCamelCase__) as cl:
									__lowerCamelCase				:				Tuple					       = text_generator(UpperCamelCase__   ,max_new_tokens=1)
								self.assertNotIn(UpperCamelCase__   ,cl.out)

								with CaptureLogger(UpperCamelCase__) as cl:
									__lowerCamelCase				:				Union[str, Any]					       = text_generator(UpperCamelCase__   ,max_length=1_0)
								self.assertNotIn(UpperCamelCase__   ,cl.out)










import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def __UpperCamelCase			(       _A ,  _A  ):
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''' ,  [False, True]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A ,  keep_in_memory=_A  ).read()
						_check_json_dataset(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						_check_json_dataset(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





def __UpperCamelCase			(       _A ,  _A  ):
						# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
						lowerCAmelCase_      			=  {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
						lowerCAmelCase_      			=  features.copy()
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  features=_A ,  cache_dir=_A  ).read()
						assert isinstance(_A ,  _A  )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''split''' ,  [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A ,  split=_A  ).read()
						_check_json_dataset(_A ,  _A  )
						assert dataset.split == split if split else "train"





@pytest.mark.parametrize('''path_type''' ,  [str, list]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						if issubclass(_A ,  _A  ):
												lowerCAmelCase_      			=  jsonl_path
						elif issubclass(_A ,  _A  ):
												lowerCAmelCase_      			=  [jsonl_path]
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A  ).read()
						_check_json_dataset(_A ,  _A  )





def __UpperCamelCase			(       _A ,  _A ,  _A=("train",)  ):
						assert isinstance(_A ,  _A  )
						for split in splits:
												lowerCAmelCase_      			=  dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory''' ,  [False, True]  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												lowerCAmelCase_      			=  JsonDatasetReader({'''train''': jsonl_path} ,  cache_dir=_A ,  keep_in_memory=_A  ).read()
						_check_json_datasetdict(_A ,  _A  )





@pytest.mark.parametrize(
    '''features''' ,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  features.copy() if features else default_expected_features
						lowerCAmelCase_      			=  (
						    Features({feature: Value(_A  ) for feature, dtype in features.items()}  ) if features is not None else None
						)
						lowerCAmelCase_      			=  JsonDatasetReader({'''train''': jsonl_path} ,  features=_A ,  cache_dir=_A  ).read()
						_check_json_datasetdict(_A ,  _A  )





@pytest.mark.parametrize('''split''' ,  [None, NamedSplit('''train'''  ), '''train''', '''test''']  )
def __UpperCamelCase			(       _A ,  _A ,  _A  ):
						if split:
												lowerCAmelCase_      			=  {split: jsonl_path}
						else:
												lowerCAmelCase_      			=  '''train'''
												lowerCAmelCase_      			=  {'''train''': jsonl_path, '''test''': jsonl_path}
						lowerCAmelCase_      			=  tmp_path / '''cache'''
						lowerCAmelCase_      			=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
						lowerCAmelCase_      			=  JsonDatasetReader(_A ,  cache_dir=_A  ).read()
						_check_json_datasetdict(_A ,  _A ,  splits=list(path.keys()  )  )
						assert all(dataset[split].split == split for split in path.keys()  )





def __UpperCamelCase			(       _A  ):
						return json.load(_A  )





def __UpperCamelCase			(       _A  ):
						return [json.loads(_A  ) for line in buffer]






class  A		:



		@pytest.mark.parametrize('''lines, load_json_function''',	[(True, load_json_lines), (False, load_json)]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json_function(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								assert isinstance(exported_content[0],	UpperCamelCase__   )
								assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at''',	[
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''   ), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ],	)
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	orient=UpperCamelCase__   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(UpperCamelCase__,	'''keys'''   ) and not hasattr(exported_content[0],	'''keys'''   )
								if len_at:
														assert len(exported_content[len_at]   ) == 10
								else:
														assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize('''lines, load_json_function''',	[(True, load_json_lines), (False, load_json)]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	num_proc=2   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json_function(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								assert isinstance(exported_content[0],	UpperCamelCase__   )
								assert len(UpperCamelCase__   ) == 10



		@pytest.mark.parametrize(
		    '''orient, container, keys, len_at''',	[
		        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
		        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
		        ('''index''', dict, set('''0123456789'''   ), None),
		        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
		        ('''values''', list, None, None),
		        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
		    ],	)
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								with io.BytesIO() as buffer:
														JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	lines=UpperCamelCase__,	orient=UpperCamelCase__,	num_proc=2   ).write()
														buffer.seek(0   )
														lowerCAmelCase_      			=  load_json(UpperCamelCase__   )
								assert isinstance(UpperCamelCase__,	UpperCamelCase__   )
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(UpperCamelCase__,	'''keys'''   ) and not hasattr(exported_content[0],	'''keys'''   )
								if len_at:
														assert len(exported_content[len_at]   ) == 10
								else:
														assert len(UpperCamelCase__   ) == 10



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								with pytest.raises(UpperCamelCase__   ):
														with io.BytesIO() as buffer:
																				JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	num_proc=0   )







		@pytest.mark.parametrize('''compression, extension''',	[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]   )
		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  tmp_path_factory.mktemp('''data'''   ) / f"test.json.{extension}"
								lowerCAmelCase_      			=  str(shared_datadir / f"test_file.json.{extension}"   )
								JsonDatasetWriter(UpperCamelCase__,	UpperCamelCase__,	compression=UpperCamelCase__   ).write()

								with fsspec.open(UpperCamelCase__,	'''rb''',	compression='''infer'''   ) as f:
														lowerCAmelCase_      			=  f.read()
								with fsspec.open(UpperCamelCase__,	'''rb''',	compression='''infer'''   ) as f:
														lowerCAmelCase_      			=  f.read()
								assert exported_content == original_content




