
import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil







class      A__							(     __magic_name__	):






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence([1, 2, 3]   )   )
							self.assertEqual(arr.type    ,     pa.intaa()   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							with self.assertRaises(a   ):
										lowerCAmelCase__      :      str         =		pa.array(TypedSequence([1, 2, 3]   )    ,     type=pa.intaa()   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							with self.assertRaises(a   ):
										lowerCAmelCase__      :      str         =		pa.array(TypedSequence([1, 2, 3]    ,     try_type=Value('bool'   )    ,     type=Value('int64'   )   )   )






				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence([1, 2, 3]    ,     type=Value('int32'   )   )   )
							self.assertEqual(arr.type    ,     pa.intaa()   )






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
										lowerCAmelCase__      :      Any         =		pa.array(TypedSequence(['foo', 'bar']    ,     type=Value('int64'   )   )   )






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		pa.array(TypedSequence([1, 2, 3]    ,     try_type=Value('int32'   )   )   )
							self.assertEqual(arr.type    ,     pa.intaa()   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence(['foo', 'bar']    ,     try_type=Value('int64'   )   )   )
							self.assertEqual(arr.type    ,     pa.string()   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence([[[1, 2, 3]]]    ,     type=ArrayaD((1, 3)    ,     'int64'   )   )   )
							self.assertEqual(arr.type    ,     ArrayaDExtensionType((1, 3)    ,     'int64'   )   )






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
										lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence(['foo', 'bar']    ,     type=ArrayaD((1, 3)    ,     'int64'   )   )   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		pa.array(TypedSequence([[[1, 2, 3]]]    ,     try_type=ArrayaD((1, 3)    ,     'int64'   )   )   )
							self.assertEqual(arr.type    ,     ArrayaDExtensionType((1, 3)    ,     'int64'   )   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		pa.array(TypedSequence(['foo', 'bar']    ,     try_type=ArrayaD((1, 3)    ,     'int64'   )   )   )
							self.assertEqual(arr.type    ,     pa.string()   )






				@require_pil
				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							import PIL.Image

							lowerCAmelCase__      :      List[str]         =		PIL.Image.fromarray(np.arange(10    ,     dtype=np.uinta   ).reshape(2    ,     5   )   )
							with patch(
							    'datasets.arrow_writer.cast_to_python_objects'    ,     side_effect=a   ) as mock_cast_to_python_objects:
										lowerCAmelCase__      :      Tuple         =		pa.array(TypedSequence([{'path': None, 'bytes': b'image_bytes'}, pil_image]    ,     type=Image()   )   )
										lowerCAmelCase__      :      List[str]         =		mock_cast_to_python_objects.call_args_list[-1]
										self.assertIn('optimize_list_casting'    ,     a   )
										self.assertFalse(kwargs['optimize_list_casting']   )






def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		str:
			lowerCAmelCase__      :      List[Any]         =		pa.BufferReader(_UpperCamelCase     ) if isinstance(_UpperCamelCase       ,							pa.Buffer     ) else pa.memory_map(_UpperCamelCase     )
			lowerCAmelCase__      :      Optional[int]         =		pa.ipc.open_stream(_UpperCamelCase     )
			lowerCAmelCase__      :      pa.Table         =		f.read_all()
			assert len(pa_table.to_batches()     ) == expected_num_chunks
			assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
			del pa_table
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 1, 10]     )
@pytest.mark.parametrize(
    'fields'       ,							[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Tuple:
			lowerCAmelCase__      :      str         =		pa.BufferOutputStream()
			lowerCAmelCase__      :      List[Any]         =		pa.schema(_UpperCamelCase     ) if fields else None
			with ArrowWriter(stream=_UpperCamelCase       ,							schema=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase     ) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}     )
						writer.write({'col_1': 'bar', 'col_2': 2}     )
						lowerCAmelCase__      :      Tuple         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						lowerCAmelCase__      :      Optional[Any]         =		{"""col_1""": pa.string(), """col_2""": pa.intaa()}
			assert writer._schema == pa.schema(_UpperCamelCase       ,							metadata=writer._schema.metadata     )
			_check_output(output.getvalue()       ,							expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )
def  lowerCAmelCase__			(  )				->		Optional[Any]:
			lowerCAmelCase__      :      List[Any]         =		pa.BufferOutputStream()
			lowerCAmelCase__      :      Any         =		Features({'labels': ClassLabel(names=['neg', 'pos']     )}     )
			with ArrowWriter(stream=_UpperCamelCase       ,							features=_UpperCamelCase     ) as writer:
						writer.write({'labels': 0}     )
						writer.write({'labels': 1}     )
						lowerCAmelCase__      :      str         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			assert writer._schema == features.arrow_schema
			assert writer._schema.metadata == features.arrow_schema.metadata
			lowerCAmelCase__      :      Optional[Any]         =		pa.BufferReader(output.getvalue()     )
			lowerCAmelCase__      :      int         =		pa.ipc.open_stream(_UpperCamelCase     )
			lowerCAmelCase__      :      pa.Table         =		f.read_all()
			lowerCAmelCase__      :      Tuple         =		pa_table.schema
			assert pa_table.num_rows == 2
			assert schema == features.arrow_schema
			assert schema.metadata == features.arrow_schema.metadata
			assert features == Features.from_arrow_schema(_UpperCamelCase     )
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 1, 10]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Union[str, Any]:
			lowerCAmelCase__      :      int         =		pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase       ,							hash_salt='split_name'       ,							check_duplicates=_UpperCamelCase       ,							) as writer:
						with pytest.raises(_UpperCamelCase     ):
									writer.write({'col_1': 'foo', 'col_2': 1}       ,							key=[1, 2]     )
									lowerCAmelCase__      :      List[str]         =		writer.finalize()
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 2, 10]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		List[str]:
			lowerCAmelCase__      :      Optional[Any]         =		pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase       ,							hash_salt='split_name'       ,							check_duplicates=_UpperCamelCase       ,							) as writer:
						with pytest.raises(_UpperCamelCase     ):
									writer.write({'col_1': 'foo', 'col_2': 1}       ,							key=10     )
									writer.write({'col_1': 'bar', 'col_2': 2}       ,							key=10     )
									lowerCAmelCase__      :      List[Any]         =		writer.finalize()
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 2, 10]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Dict:
			lowerCAmelCase__      :      str         =		pa.BufferOutputStream()
			with ArrowWriter(
			    stream=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase       ,							hash_salt='split_name'       ,							check_duplicates=_UpperCamelCase       ,							) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}       ,							key=1     )
						writer.write({'col_1': 'bar', 'col_2': 2}       ,							key=2     )
						lowerCAmelCase__      :      List[Any]         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			_check_output(output.getvalue()       ,							expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 1, 10]     )
@pytest.mark.parametrize(
    'fields'       ,							[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Dict:
			lowerCAmelCase__      :      Tuple         =		pa.BufferOutputStream()
			lowerCAmelCase__      :      List[Any]         =		pa.schema(_UpperCamelCase     ) if fields else None
			with ArrowWriter(stream=_UpperCamelCase       ,							schema=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase     ) as writer:
						writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )
						writer.write_batch({'col_1': [], 'col_2': []}     )
						lowerCAmelCase__      :      List[Any]         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						lowerCAmelCase__      :      Dict         =		{"""col_1""": pa.string(), """col_2""": pa.intaa()}
			assert writer._schema == pa.schema(_UpperCamelCase       ,							metadata=writer._schema.metadata     )
			_check_output(output.getvalue()       ,							expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 1, 10]     )
@pytest.mark.parametrize(
    'fields'       ,							[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Tuple:
			lowerCAmelCase__      :      Optional[int]         =		pa.BufferOutputStream()
			lowerCAmelCase__      :      Optional[Any]         =		pa.schema(_UpperCamelCase     ) if fields else None
			with ArrowWriter(stream=_UpperCamelCase       ,							schema=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase     ) as writer:
						writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )     )
						lowerCAmelCase__      :      str         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						lowerCAmelCase__      :      int         =		{"""col_1""": pa.string(), """col_2""": pa.intaa()}
			assert writer._schema == pa.schema(_UpperCamelCase       ,							metadata=writer._schema.metadata     )
			_check_output(output.getvalue()       ,							expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )
@pytest.mark.parametrize('writer_batch_size'       ,							[None, 1, 10]     )
@pytest.mark.parametrize(
    'fields'       ,							[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Any:
			lowerCAmelCase__      :      List[str]         =		pa.BufferOutputStream()
			lowerCAmelCase__      :      Union[str, Any]         =		pa.schema(_UpperCamelCase     ) if fields else None
			with ArrowWriter(stream=_UpperCamelCase       ,							schema=_UpperCamelCase       ,							writer_batch_size=_UpperCamelCase     ) as writer:
						writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}     )     )
						writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}     )     )
						lowerCAmelCase__      :      List[Any]         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			if not fields:
						lowerCAmelCase__      :      Optional[Any]         =		{"""col_1""": pa.string(), """col_2""": pa.intaa()}
			assert writer._schema == pa.schema(_UpperCamelCase       ,							metadata=writer._schema.metadata     )
			_check_output(output.getvalue()       ,							expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )
def  lowerCAmelCase__			(  )				->		Optional[Any]:



			with tempfile.TemporaryDirectory() as tmp_dir:
						lowerCAmelCase__      :      int         =		{"""col_1""": pa.string(), """col_2""": pa.intaa()}
						lowerCAmelCase__      :      List[Any]         =		os.path.join(_UpperCamelCase       ,							'test.arrow'     )
						with ArrowWriter(path=_UpperCamelCase       ,							schema=pa.schema(_UpperCamelCase     )     ) as writer:
									writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )
									lowerCAmelCase__      :      List[Any]         =		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == pa.schema(_UpperCamelCase       ,							metadata=writer._schema.metadata     )
						_check_output(_UpperCamelCase       ,							1     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Any:



			if pa.types.is_list(_UpperCamelCase     ):
						return get_base_dtype(arr_type.value_type     )
			else:
						return arr_type
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Optional[int]:



			if isinstance(lst[0]       ,							_UpperCamelCase     ):
						change_first_primitive_element_in_list(lst[0]       ,							_UpperCamelCase     )
			else:
						lowerCAmelCase__      :      List[Any]         =		value
@pytest.mark.parametrize('optimized_int_type, expected_dtype'       ,							[(None, pa.intaa()), (Value('int32'     ), pa.intaa())]     )
@pytest.mark.parametrize('sequence'       ,							[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		str:
			lowerCAmelCase__      :      Optional[Any]         =		pa.array(TypedSequence(_UpperCamelCase       ,							optimized_int_type=_UpperCamelCase     )     )
			assert get_base_dtype(arr.type     ) == expected_dtype
@pytest.mark.parametrize(
    'col, expected_dtype'       ,							[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]       ,							)
@pytest.mark.parametrize('sequence'       ,							[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		List[str]:
			lowerCAmelCase__      :      Union[str, Any]         =		pa.array(OptimizedTypedSequence(_UpperCamelCase       ,							col=_UpperCamelCase     )     )
			assert get_base_dtype(arr.type     ) == expected_dtype

			# not in range
			if col != "other":
						# avoids errors due to in-place modifications
						lowerCAmelCase__      :      Optional[Any]         =		copy.deepcopy(_UpperCamelCase     )
						lowerCAmelCase__      :      Optional[int]         =		np.iinfo(expected_dtype.to_pandas_dtype()     ).max + 1
						change_first_primitive_element_in_list(_UpperCamelCase       ,							_UpperCamelCase     )
						lowerCAmelCase__      :      Any         =		pa.array(OptimizedTypedSequence(_UpperCamelCase       ,							col=_UpperCamelCase     )     )
						assert get_base_dtype(arr.type     ) == pa.intaa()
@pytest.mark.parametrize('raise_exception'       ,							[False, True]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		List[Any]:
			lowerCAmelCase__      :      Optional[Any]         =		str(tmp_path / 'dataset-train.arrow'     )
			try:
						with ArrowWriter(path=_UpperCamelCase     ) as writer:
									if raise_exception:
												raise pa.lib.ArrowInvalid()
									else:
												writer.stream.close()
			except pa.lib.ArrowInvalid:
						pass
			finally:
						assert writer.stream.closed
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Dict:
			lowerCAmelCase__      :      Tuple         =		"""mock://dataset-train.arrow"""
			with ArrowWriter(path=_UpperCamelCase       ,							storage_options=mockfs.storage_options     ) as writer:
						assert isinstance(writer._fs       ,							type(_UpperCamelCase     )     )
						assert writer._fs.storage_options == mockfs.storage_options
						writer.write({'col_1': 'foo', 'col_2': 1}     )
						writer.write({'col_1': 'bar', 'col_2': 2}     )
						lowerCAmelCase__      :      List[Any]         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			assert mockfs.exists(_UpperCamelCase     )
def  lowerCAmelCase__			(  )				->		Optional[int]:
			lowerCAmelCase__      :      Any         =		pa.BufferOutputStream()
			with ParquetWriter(stream=_UpperCamelCase     ) as writer:
						writer.write({'col_1': 'foo', 'col_2': 1}     )
						writer.write({'col_1': 'bar', 'col_2': 2}     )
						lowerCAmelCase__      :      Any         =		writer.finalize()
			assert num_examples == 2
			assert num_bytes > 0
			lowerCAmelCase__      :      int         =		pa.BufferReader(output.getvalue()     )
			lowerCAmelCase__      :      pa.Table         =		pq.read_table(_UpperCamelCase     )
			assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize('embed_local_files'       ,							[False, True]     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Optional[int]:
			import PIL.Image

			lowerCAmelCase__      :      List[Any]         =		str(tmp_path / 'test_image_rgb.jpg'     )
			PIL.Image.fromarray(np.zeros((5, 5)       ,							dtype=np.uinta     )     ).save(_UpperCamelCase       ,							format='png'     )
			lowerCAmelCase__      :      Tuple         =		pa.BufferOutputStream()
			with ParquetWriter(
			    stream=_UpperCamelCase       ,							features=Features({'image': Image()}     )       ,							embed_local_files=_UpperCamelCase     ) as writer:
						writer.write({'image': image_path}     )
						writer.finalize()
			lowerCAmelCase__      :      str         =		pa.BufferReader(output.getvalue()     )
			lowerCAmelCase__      :      pa.Table         =		pq.read_table(_UpperCamelCase     )
			lowerCAmelCase__      :      Dict         =		pa_table.to_pydict()
			if embed_local_files:
						assert isinstance(out['image'][0]['path']       ,							_UpperCamelCase     )
						with open(_UpperCamelCase       ,							'rb'     ) as f:
									assert out["image"][0]["bytes"] == f.read()
			else:
						assert out["image"][0]["path"] == image_path
						assert out["image"][0]["bytes"] is None







def  lowerCAmelCase__			(  )				->		int:
			lowerCAmelCase__      :      Tuple         =		pa.schema([pa.field('col_1'       ,							pa.string()       ,							nullable=_UpperCamelCase     )]     )
			lowerCAmelCase__      :      Union[str, Any]         =		pa.BufferOutputStream()
			with ArrowWriter(stream=_UpperCamelCase     ) as writer:
						writer._build_writer(inferred_schema=_UpperCamelCase     )
						assert writer._schema == pa.schema([pa.field('col_1'       ,							pa.string()     )]     )



"""simple docstring"""






def      lowerCamelCase       (     _UpperCamelCase		:      int = 4_0_0_0_0_0_0							)		->       int:

							'''simple docstring'''



							__UpperCAmelCase   :						int      							=					[0, 1]
							__UpperCAmelCase   :						Optional[Any]      							=					0
							while fib[i] <= n:
														fib.append(fib[i] + fib[i + 1]							)
														if fib[i + 2] > n:
																					break
														i += 1
							__UpperCAmelCase   :						str      							=					0
							for j in range(len(_UpperCamelCase							) - 1							):
														if fib[j] % 2 == 0:
																					total += fib[j]

							return total


if __name__ == "__main__":
							print(F"{solution() = }")
