



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil







class 			__lowerCAmelCase    (			lowerCAmelCase_						):



			def     _lowerCamelCase       (						self   :  List[Any])   ->    str:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence([1, 2, 3]))
						self.assertEqual(arr.type ,		pa.intaa())



			def     _lowerCamelCase       (						self   :  Dict)   ->    str:



						"""simple docstring"""






						with self.assertRaises(_snake_case):
									_UpperCAmelCase		       =   pa.array(TypedSequence([1, 2, 3]) ,		type=pa.intaa())



			def     _lowerCamelCase       (						self   :  List[Any])   ->    Dict:



						"""simple docstring"""






						with self.assertRaises(_snake_case):
									_UpperCAmelCase		       =   pa.array(TypedSequence([1, 2, 3] ,		try_type=Value('bool') ,		type=Value('int64')))



			def     _lowerCamelCase       (						self   :  int)   ->    Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence([1, 2, 3] ,		type=Value('int32')))
						self.assertEqual(arr.type ,		pa.intaa())



			def     _lowerCamelCase       (						self   :  List[str])   ->    Optional[Any]:



						"""simple docstring"""






						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
									_UpperCAmelCase		       =   pa.array(TypedSequence(['foo', 'bar'] ,		type=Value('int64')))



			def     _lowerCamelCase       (						self   :  Tuple)   ->    Any:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence([1, 2, 3] ,		try_type=Value('int32')))
						self.assertEqual(arr.type ,		pa.intaa())



			def     _lowerCamelCase       (						self   :  Optional[int])   ->    List[str]:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence(['foo', 'bar'] ,		try_type=Value('int64')))
						self.assertEqual(arr.type ,		pa.string())



			def     _lowerCamelCase       (						self   :  str)   ->    List[Any]:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence([[[1, 2, 3]]] ,		type=ArrayaD((1, 3) ,		'int64')))
						self.assertEqual(arr.type ,		ArrayaDExtensionType((1, 3) ,		'int64'))



			def     _lowerCamelCase       (						self   :  Any)   ->    List[Any]:



						"""simple docstring"""






						with self.assertRaises((TypeError, pa.lib.ArrowInvalid)):
									_UpperCAmelCase		       =   pa.array(TypedSequence(['foo', 'bar'] ,		type=ArrayaD((1, 3) ,		'int64')))



			def     _lowerCamelCase       (						self   :  Union[str, Any])   ->    Tuple:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence([[[1, 2, 3]]] ,		try_type=ArrayaD((1, 3) ,		'int64')))
						self.assertEqual(arr.type ,		ArrayaDExtensionType((1, 3) ,		'int64'))



			def     _lowerCamelCase       (						self   :  int)   ->    Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase		       =   pa.array(TypedSequence(['foo', 'bar'] ,		try_type=ArrayaD((1, 3) ,		'int64')))
						self.assertEqual(arr.type ,		pa.string())



			@require_pil
			def     _lowerCamelCase       (						self   :  Any)   ->    Optional[int]:



						"""simple docstring"""






						import PIL.Image

						_UpperCAmelCase		       =   PIL.Image.fromarray(np.arange(10 ,		dtype=np.uinta).reshape(2 ,		5))
						with patch(
						    'datasets.arrow_writer.cast_to_python_objects' ,		side_effect=_snake_case) as mock_cast_to_python_objects:
									_UpperCAmelCase		       =   pa.array(TypedSequence([{'path': None, 'bytes': B'image_bytes'}, pil_image] ,		type=Image()))
									_UpperCAmelCase		       =   mock_cast_to_python_objects.call_args_list[-1]
									self.assertIn('optimize_list_casting' ,		_snake_case)
									self.assertFalse(kwargs['optimize_list_casting'])





def 							A	(     _UpperCAmelCase				:   Dict							,   _UpperCAmelCase				:   int    )				-> Dict:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferReader(_UpperCAmelCase    ) if isinstance(_UpperCAmelCase							,   pa.Buffer    ) else pa.memory_map(_UpperCAmelCase    )
			_UpperCAmelCase		       =   pa.ipc.open_stream(_UpperCAmelCase    )
			_UpperCAmelCase		       =   f.read_all()
			assert len(pa_table.to_batches()    ) == expected_num_chunks
			assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
			del pa_table





@pytest.mark.parametrize('writer_batch_size'							,   [None, 1, 10]    )
@pytest.mark.parametrize(
    'fields'							,   [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]    )
def 							A	(     _UpperCAmelCase				:   List[str]							,   _UpperCAmelCase				:   List[str]    )				-> List[Any]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			_UpperCAmelCase		       =   pa.schema(_UpperCAmelCase    ) if fields else None
			with ArrowWriter(stream=_UpperCAmelCase							,   schema=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase    ) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}    )
						writer.write({'col_1': 'bar', 'col_2': 2}    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						_UpperCAmelCase		       =   {'col_1': pa.string(), 'col_2': pa.intaa()}
			assert writer._schema == pa.schema(_UpperCAmelCase							,   metadata=writer._schema.metadata    )
			_check_output(output.getvalue()							,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )





def 							A	(     )				-> Optional[int]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			_UpperCAmelCase		       =   Features({'labels': ClassLabel(names=['neg', 'pos']    )}    )
			with ArrowWriter(stream=_UpperCAmelCase							,   features=_UpperCAmelCase    ) as writer:
						writer.write({'labels': 0}    )
						writer.write({'labels': 1}    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			assert writer._schema == features.arrow_schema
			assert writer._schema.metadata == features.arrow_schema.metadata
			_UpperCAmelCase		       =   pa.BufferReader(output.getvalue()    )
			_UpperCAmelCase		       =   pa.ipc.open_stream(_UpperCAmelCase    )
			_UpperCAmelCase		       =   f.read_all()
			_UpperCAmelCase		       =   pa_table.schema
			assert pa_table.num_rows == 2
			assert schema == features.arrow_schema
			assert schema.metadata == features.arrow_schema.metadata
			assert features == Features.from_arrow_schema(_UpperCAmelCase    )





@pytest.mark.parametrize('writer_batch_size'							,   [None, 1, 10]    )
def 							A	(     _UpperCAmelCase				:   int    )				-> Dict:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase							,   hash_salt='split_name'							,   check_duplicates=_UpperCAmelCase							,   ) as writer:
						with pytest.raises(_UpperCAmelCase    ):
									writer.write({'col_1': 'foo', 'col_2': 1}							,   key=[1, 2]    )
									_UpperCAmelCase		       =   writer.finalize()





@pytest.mark.parametrize('writer_batch_size'							,   [None, 2, 10]    )
def 							A	(     _UpperCAmelCase				:   Optional[int]    )				-> List[Any]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase							,   hash_salt='split_name'							,   check_duplicates=_UpperCAmelCase							,   ) as writer:
						with pytest.raises(_UpperCAmelCase    ):
									writer.write({'col_1': 'foo', 'col_2': 1}							,   key=10    )
									writer.write({'col_1': 'bar', 'col_2': 2}							,   key=10    )
									_UpperCAmelCase		       =   writer.finalize()





@pytest.mark.parametrize('writer_batch_size'							,   [None, 2, 10]    )
def 							A	(     _UpperCAmelCase				:   List[Any]    )				-> Optional[int]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase							,   hash_salt='split_name'							,   check_duplicates=_UpperCAmelCase							,   ) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}							,   key=1    )
						writer.write({'col_1': 'bar', 'col_2': 2}							,   key=2    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			_check_output(output.getvalue()							,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )





@pytest.mark.parametrize('writer_batch_size'							,   [None, 1, 10]    )
@pytest.mark.parametrize(
    'fields'							,   [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]    )
def 							A	(     _UpperCAmelCase				:   Dict							,   _UpperCAmelCase				:   Any    )				-> List[Any]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			_UpperCAmelCase		       =   pa.schema(_UpperCAmelCase    ) if fields else None
			with ArrowWriter(stream=_UpperCAmelCase							,   schema=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase    ) as writer:
						writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}    )
						writer.write_batch({'col_1': [], 'col_2': []}    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						_UpperCAmelCase		       =   {'col_1': pa.string(), 'col_2': pa.intaa()}
			assert writer._schema == pa.schema(_UpperCAmelCase							,   metadata=writer._schema.metadata    )
			_check_output(output.getvalue()							,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )





@pytest.mark.parametrize('writer_batch_size'							,   [None, 1, 10]    )
@pytest.mark.parametrize(
    'fields'							,   [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]    )
def 							A	(     _UpperCAmelCase				:   Tuple							,   _UpperCAmelCase				:   str    )				-> Any:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			_UpperCAmelCase		       =   pa.schema(_UpperCAmelCase    ) if fields else None
			with ArrowWriter(stream=_UpperCAmelCase							,   schema=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase    ) as writer:
						writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}    )    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						_UpperCAmelCase		       =   {'col_1': pa.string(), 'col_2': pa.intaa()}
			assert writer._schema == pa.schema(_UpperCAmelCase							,   metadata=writer._schema.metadata    )
			_check_output(output.getvalue()							,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )





@pytest.mark.parametrize('writer_batch_size'							,   [None, 1, 10]    )
@pytest.mark.parametrize(
    'fields'							,   [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]    )
def 							A	(     _UpperCAmelCase				:   List[str]							,   _UpperCAmelCase				:   Union[str, Any]    )				-> Tuple:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			_UpperCAmelCase		       =   pa.schema(_UpperCAmelCase    ) if fields else None
			with ArrowWriter(stream=_UpperCAmelCase							,   schema=_UpperCAmelCase							,   writer_batch_size=_UpperCAmelCase    ) as writer:
						writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}    )    )
						writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}    )    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						_UpperCAmelCase		       =   {'col_1': pa.string(), 'col_2': pa.intaa()}
			assert writer._schema == pa.schema(_UpperCAmelCase							,   metadata=writer._schema.metadata    )
			_check_output(output.getvalue()							,   expected_num_chunks=num_examples if writer_batch_size == 1 else 1    )





def 							A	(     )				-> int:







			'''simple docstring'''
			with tempfile.TemporaryDirectory() as tmp_dir:
						_UpperCAmelCase		       =   {'col_1': pa.string(), 'col_2': pa.intaa()}
						_UpperCAmelCase		       =   os.path.join(_UpperCAmelCase							,   'test.arrow'    )
						with ArrowWriter(path=_UpperCAmelCase							,   schema=pa.schema(_UpperCAmelCase    )    ) as writer:
									writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}    )
									_UpperCAmelCase		       =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == pa.schema(_UpperCAmelCase							,   metadata=writer._schema.metadata    )
						_check_output(_UpperCAmelCase							,   1    )





def 							A	(     _UpperCAmelCase				:   Union[str, Any]    )				-> Dict:







			'''simple docstring'''
			if pa.types.is_list(_UpperCAmelCase    ):
						return get_base_dtype(arr_type.value_type    )
			else:
						return arr_type





def 							A	(     _UpperCAmelCase				:   List[Any]							,   _UpperCAmelCase				:   Union[str, Any]    )				-> Union[str, Any]:







			'''simple docstring'''
			if isinstance(lst[0]							,   _UpperCAmelCase    ):
						change_first_primitive_element_in_list(lst[0]							,   _UpperCAmelCase    )
			else:
						_UpperCAmelCase		       =   value





@pytest.mark.parametrize('optimized_int_type, expected_dtype'							,   [(None, pa.intaa()), (Value('int32'    ), pa.intaa())]    )
@pytest.mark.parametrize('sequence'							,   [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 							A	(     _UpperCAmelCase				:   Tuple							,   _UpperCAmelCase				:   List[str]							,   _UpperCAmelCase				:   List[Any]    )				-> Union[str, Any]:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.array(TypedSequence(_UpperCAmelCase							,   optimized_int_type=_UpperCAmelCase    )    )
			assert get_base_dtype(arr.type    ) == expected_dtype





@pytest.mark.parametrize(
    'col, expected_dtype'							,   [
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]							,   )
@pytest.mark.parametrize('sequence'							,   [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]    )
def 							A	(     _UpperCAmelCase				:   List[Any]							,   _UpperCAmelCase				:   Union[str, Any]							,   _UpperCAmelCase				:   str    )				-> Optional[Any]:







			'''simple docstring'''
			# in range
			_UpperCAmelCase		       =   pa.array(OptimizedTypedSequence(_UpperCAmelCase							,   col=_UpperCAmelCase    )    )
			assert get_base_dtype(arr.type    ) == expected_dtype

			# not in range
			if col != "other":
						# avoids errors due to in-place modifications
						_UpperCAmelCase		       =   copy.deepcopy(_UpperCAmelCase    )
						_UpperCAmelCase		       =   np.iinfo(expected_dtype.to_pandas_dtype()    ).max + 1
						change_first_primitive_element_in_list(_UpperCAmelCase							,   _UpperCAmelCase    )
						_UpperCAmelCase		       =   pa.array(OptimizedTypedSequence(_UpperCAmelCase							,   col=_UpperCAmelCase    )    )
						assert get_base_dtype(arr.type    ) == pa.intaa()





@pytest.mark.parametrize('raise_exception'							,   [False, True]    )
def 							A	(     _UpperCAmelCase				:   Union[str, Any]							,   _UpperCAmelCase				:   List[str]    )				-> Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase		       =   str(tmp_path / 'dataset-train.arrow'    )
			try:
						with ArrowWriter(path=_UpperCAmelCase    ) as writer:
									if raise_exception:
												raise pa.lib.ArrowInvalid()
									else:
												writer.stream.close()
			except pa.lib.ArrowInvalid:
						pass
			finally:
						assert writer.stream.closed





def 							A	(     _UpperCAmelCase				:   int    )				-> Tuple:







			'''simple docstring'''
			_UpperCAmelCase		       =   'mock://dataset-train.arrow'
			with ArrowWriter(path=_UpperCAmelCase							,   storage_options=mockfs.storage_options    ) as writer:
						assert isinstance(writer._fs							,   type(_UpperCAmelCase    )    )
						assert writer._fs.storage_options == mockfs.storage_options
						writer.write({'col_1': 'foo', 'col_2': 1}    )
						writer.write({'col_1': 'bar', 'col_2': 2}    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			assert mockfs.exists(_UpperCAmelCase    )





def 							A	(     )				-> Any:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ParquetWriter(stream=_UpperCAmelCase    ) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}    )
						writer.write({'col_1': 'bar', 'col_2': 2}    )
						_UpperCAmelCase		       =   writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			_UpperCAmelCase		       =   pa.BufferReader(output.getvalue()    )
			_UpperCAmelCase		       =   pq.read_table(_UpperCAmelCase    )
			assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}





@require_pil
@pytest.mark.parametrize('embed_local_files'							,   [False, True]    )
def 							A	(     _UpperCAmelCase				:   List[Any]							,   _UpperCAmelCase				:   Optional[Any]    )				-> List[Any]:







			'''simple docstring'''
			import PIL.Image

			_UpperCAmelCase		       =   str(tmp_path / 'test_image_rgb.jpg'    )
			PIL.Image.fromarray(np.zeros((5, 5)							,   dtype=np.uinta    )    ).save(_UpperCAmelCase							,   format='png'    )
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ParquetWriter(
			    stream=_UpperCAmelCase							,   features=Features({'image': Image()}    )							,   embed_local_files=_UpperCAmelCase    ) as writer:
						writer.write({'image': image_path}    )
						writer.finalize()
			_UpperCAmelCase		       =   pa.BufferReader(output.getvalue()    )
			_UpperCAmelCase		       =   pq.read_table(_UpperCAmelCase    )
			_UpperCAmelCase		       =   pa_table.to_pydict()
			if embed_local_files:
						assert isinstance(out['image'][0]['path']							,   _UpperCAmelCase    )
						with open(_UpperCAmelCase							,   'rb'    ) as f:
									assert out["image"][0]["bytes"] == f.read()
			else:
						assert out["image"][0]["path"] == image_path
						assert out["image"][0]["bytes"] is None





def 							A	(     )				-> Dict:







			'''simple docstring'''
			_UpperCAmelCase		       =   pa.schema([pa.field('col_1'							,   pa.string()							,   nullable=_UpperCAmelCase    )]    )
			_UpperCAmelCase		       =   pa.BufferOutputStream()
			with ArrowWriter(stream=_UpperCAmelCase    ) as writer:
						writer._build_writer(inferred_schema=_UpperCAmelCase    )
						assert writer._schema == pa.schema([pa.field('col_1'							,   pa.string()    )]    )





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool




class 					snake_case__						(       lowerCAmelCase_     ):






     """simple docstring"""


     _SCREAMING_SNAKE_CASE						=	"""philschmid/bart-large-cnn-samsum"""
     _SCREAMING_SNAKE_CASE						=	(
         """This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, """
         """and returns a summary of the text."""
     )
     _SCREAMING_SNAKE_CASE						=	"""summarizer"""
     _SCREAMING_SNAKE_CASE						=	AutoTokenizer
     _SCREAMING_SNAKE_CASE						=	AutoModelForSeqaSeqLM

     _SCREAMING_SNAKE_CASE						=	["""text"""]
     _SCREAMING_SNAKE_CASE						=	["""text"""]






     def    lowercase_   (       self  :  Optional[Any], _snake_case  :  str					)   ->Any:
            return self.pre_processor(_snake_case, return_tensors='pt', truncation=_snake_case					)






     def    lowercase_   (       self  :  int, _snake_case  :  List[Any]					)   ->Any:
            return self.model.generate(**_snake_case					)[0]






     def    lowercase_   (       self  :  int, _snake_case  :  int					)   ->str:
            return self.pre_processor.decode(_snake_case, skip_special_tokens=_snake_case, clean_up_tokenization_spaces=_snake_case					)






