



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil




class 					snake_case__						(       lowerCAmelCase_     ):






     """simple docstring"""


     def    lowercase_   (       self  :  List[Any]					)   ->str:
            snake_case__							:	List[str]   	=	pa.array(TypedSequence([1, 2, 3]					)					)
            self.assertEqual(arr.type, pa.intaa()					)






     def    lowercase_   (       self  :  Dict					)   ->str:
            with self.assertRaises(_snake_case					):
                   snake_case__							:	Optional[int]   	=	pa.array(TypedSequence([1, 2, 3]					), type=pa.intaa()					)






     def    lowercase_   (       self  :  List[Any]					)   ->Dict:
            with self.assertRaises(_snake_case					):
                   snake_case__							:	Any   	=	pa.array(TypedSequence([1, 2, 3], try_type=Value('bool'					), type=Value('int64'					)					)					)






     def    lowercase_   (       self  :  int					)   ->Optional[int]:
            snake_case__							:	Union[str, Any]   	=	pa.array(TypedSequence([1, 2, 3], type=Value('int32'					)					)					)
            self.assertEqual(arr.type, pa.intaa()					)






     def    lowercase_   (       self  :  List[str]					)   ->Optional[Any]:
            with self.assertRaises((TypeError, pa.lib.ArrowInvalid)					):
                   snake_case__							:	int   	=	pa.array(TypedSequence(['foo', 'bar'], type=Value('int64'					)					)					)






     def    lowercase_   (       self  :  Tuple					)   ->Any:
            snake_case__							:	List[str]   	=	pa.array(TypedSequence([1, 2, 3], try_type=Value('int32'					)					)					)
            self.assertEqual(arr.type, pa.intaa()					)






     def    lowercase_   (       self  :  Optional[int]					)   ->List[str]:
            snake_case__							:	str   	=	pa.array(TypedSequence(['foo', 'bar'], try_type=Value('int64'					)					)					)
            self.assertEqual(arr.type, pa.string()					)






     def    lowercase_   (       self  :  str					)   ->List[Any]:
            snake_case__							:	Tuple   	=	pa.array(TypedSequence([[[1, 2, 3]]], type=ArrayaD((1, 3), 'int64'					)					)					)
            self.assertEqual(arr.type, ArrayaDExtensionType((1, 3), 'int64'					)					)






     def    lowercase_   (       self  :  Any					)   ->List[Any]:
            with self.assertRaises((TypeError, pa.lib.ArrowInvalid)					):
                   snake_case__							:	Any   	=	pa.array(TypedSequence(['foo', 'bar'], type=ArrayaD((1, 3), 'int64'					)					)					)






     def    lowercase_   (       self  :  Union[str, Any]					)   ->Tuple:
            snake_case__							:	Optional[Any]   	=	pa.array(TypedSequence([[[1, 2, 3]]], try_type=ArrayaD((1, 3), 'int64'					)					)					)
            self.assertEqual(arr.type, ArrayaDExtensionType((1, 3), 'int64'					)					)






     def    lowercase_   (       self  :  int					)   ->Union[str, Any]:
            snake_case__							:	int   	=	pa.array(TypedSequence(['foo', 'bar'], try_type=ArrayaD((1, 3), 'int64'					)					)					)
            self.assertEqual(arr.type, pa.string()					)






     @require_pil
     def    lowercase_   (       self  :  Any					)   ->Optional[int]:
            import PIL.Image

            snake_case__							:	Optional[int]   	=	PIL.Image.fromarray(np.arange(1_0, dtype=np.uinta					).reshape(2, 5					)					)
            with patch(
                'datasets.arrow_writer.cast_to_python_objects', side_effect=_snake_case					) as mock_cast_to_python_objects:
                   snake_case__							:	Dict   	=	pa.array(TypedSequence([{'path': None, 'bytes': b'image_bytes'}, pil_image], type=Image()					)					)
                   snake_case__	,				snake_case__							:	int   	=	mock_cast_to_python_objects.call_args_list[-1]
                   self.assertIn('optimize_list_casting', _snake_case					)
                   self.assertFalse(kwargs['optimize_list_casting']					)


def        lowercase_							(A : Dict		,					A : int      ):
       snake_case__							:	str   	=	pa.BufferReader(A      ) if isinstance(A		,					pa.Buffer      ) else pa.memory_map(A      )
       snake_case__							:	Optional[int]   	=	pa.ipc.open_stream(A      )
       snake_case__							:	pa.Table   	=	f.read_all()
       assert len(pa_table.to_batches()      ) == expected_num_chunks
       assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
       del pa_table
@pytest.mark.parametrize('writer_batch_size'		,					[None, 1, 1_0]      )
@pytest.mark.parametrize(
    'fields'		,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def        lowercase_							(A : List[str]		,					A : List[str]      ):
       snake_case__							:	Optional[Any]   	=	pa.BufferOutputStream()
       snake_case__							:	Optional[int]   	=	pa.schema(A      ) if fields else None
       with ArrowWriter(stream=A		,					schema=A		,					writer_batch_size=A      ) as writer:
              writer.write({'col_1': 'foo', 'col_2': 1}      )
              writer.write({'col_1': 'bar', 'col_2': 2}      )
              snake_case__	,				snake_case__							:	Optional[Any]   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       if not fields:
              snake_case__							:	str   	=	{'col_1': pa.string(), 'col_2': pa.intaa()}
       assert writer._schema == pa.schema(A		,					metadata=writer._schema.metadata      )
       _check_output(output.getvalue()		,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
def        lowercase_							():
       snake_case__							:	int   	=	pa.BufferOutputStream()
       snake_case__							:	Optional[int]   	=	Features({'labels': ClassLabel(names=['neg', 'pos']      )}      )
       with ArrowWriter(stream=A		,					features=A      ) as writer:
              writer.write({'labels': 0}      )
              writer.write({'labels': 1}      )
              snake_case__	,				snake_case__							:	int   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       assert writer._schema == features.arrow_schema
       assert writer._schema.metadata == features.arrow_schema.metadata
       snake_case__							:	List[str]   	=	pa.BufferReader(output.getvalue()      )
       snake_case__							:	int   	=	pa.ipc.open_stream(A      )
       snake_case__							:	pa.Table   	=	f.read_all()
       snake_case__							:	List[Any]   	=	pa_table.schema
       assert pa_table.num_rows == 2
       assert schema == features.arrow_schema
       assert schema.metadata == features.arrow_schema.metadata
       assert features == Features.from_arrow_schema(A      )
@pytest.mark.parametrize('writer_batch_size'		,					[None, 1, 1_0]      )
def        lowercase_							(A : int      ):
       snake_case__							:	str   	=	pa.BufferOutputStream()
       with ArrowWriter(
           stream=A		,					writer_batch_size=A		,					hash_salt='split_name'		,					check_duplicates=A		,					) as writer:
              with pytest.raises(A      ):
                     writer.write({'col_1': 'foo', 'col_2': 1}		,					key=[1, 2]      )
                     snake_case__	,				snake_case__							:	List[Any]   	=	writer.finalize()
@pytest.mark.parametrize('writer_batch_size'		,					[None, 2, 1_0]      )
def        lowercase_							(A : Optional[int]      ):
       snake_case__							:	Optional[int]   	=	pa.BufferOutputStream()
       with ArrowWriter(
           stream=A		,					writer_batch_size=A		,					hash_salt='split_name'		,					check_duplicates=A		,					) as writer:
              with pytest.raises(A      ):
                     writer.write({'col_1': 'foo', 'col_2': 1}		,					key=1_0      )
                     writer.write({'col_1': 'bar', 'col_2': 2}		,					key=1_0      )
                     snake_case__	,				snake_case__							:	int   	=	writer.finalize()
@pytest.mark.parametrize('writer_batch_size'		,					[None, 2, 1_0]      )
def        lowercase_							(A : List[Any]      ):
       snake_case__							:	Any   	=	pa.BufferOutputStream()
       with ArrowWriter(
           stream=A		,					writer_batch_size=A		,					hash_salt='split_name'		,					check_duplicates=A		,					) as writer:
              writer.write({'col_1': 'foo', 'col_2': 1}		,					key=1      )
              writer.write({'col_1': 'bar', 'col_2': 2}		,					key=2      )
              snake_case__	,				snake_case__							:	List[str]   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       _check_output(output.getvalue()		,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'		,					[None, 1, 1_0]      )
@pytest.mark.parametrize(
    'fields'		,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def        lowercase_							(A : Dict		,					A : Any      ):
       snake_case__							:	Union[str, Any]   	=	pa.BufferOutputStream()
       snake_case__							:	Dict   	=	pa.schema(A      ) if fields else None
       with ArrowWriter(stream=A		,					schema=A		,					writer_batch_size=A      ) as writer:
              writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )
              writer.write_batch({'col_1': [], 'col_2': []}      )
              snake_case__	,				snake_case__							:	Optional[int]   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       if not fields:
              snake_case__							:	Union[str, Any]   	=	{'col_1': pa.string(), 'col_2': pa.intaa()}
       assert writer._schema == pa.schema(A		,					metadata=writer._schema.metadata      )
       _check_output(output.getvalue()		,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'		,					[None, 1, 1_0]      )
@pytest.mark.parametrize(
    'fields'		,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def        lowercase_							(A : Tuple		,					A : str      ):
       snake_case__							:	List[str]   	=	pa.BufferOutputStream()
       snake_case__							:	int   	=	pa.schema(A      ) if fields else None
       with ArrowWriter(stream=A		,					schema=A		,					writer_batch_size=A      ) as writer:
              writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )      )
              snake_case__	,				snake_case__							:	List[Any]   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       if not fields:
              snake_case__							:	List[Any]   	=	{'col_1': pa.string(), 'col_2': pa.intaa()}
       assert writer._schema == pa.schema(A		,					metadata=writer._schema.metadata      )
       _check_output(output.getvalue()		,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
@pytest.mark.parametrize('writer_batch_size'		,					[None, 1, 1_0]      )
@pytest.mark.parametrize(
    'fields'		,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]      )
def        lowercase_							(A : List[str]		,					A : Union[str, Any]      ):
       snake_case__							:	List[Any]   	=	pa.BufferOutputStream()
       snake_case__							:	int   	=	pa.schema(A      ) if fields else None
       with ArrowWriter(stream=A		,					schema=A		,					writer_batch_size=A      ) as writer:
              writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}      )      )
              writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}      )      )
              snake_case__	,				snake_case__							:	int   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       if not fields:
              snake_case__							:	List[str]   	=	{'col_1': pa.string(), 'col_2': pa.intaa()}
       assert writer._schema == pa.schema(A		,					metadata=writer._schema.metadata      )
       _check_output(output.getvalue()		,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1      )
def        lowercase_							():
       with tempfile.TemporaryDirectory() as tmp_dir:
              snake_case__							:	int   	=	{'col_1': pa.string(), 'col_2': pa.intaa()}
              snake_case__							:	Tuple   	=	os.path.join(A		,					'test.arrow'      )
              with ArrowWriter(path=A		,					schema=pa.schema(A      )      ) as writer:
                     writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}      )
                     snake_case__	,				snake_case__							:	Dict   	=	writer.finalize()
              assert num_examples == 2
              assert num_bytes > 0
              assert writer._schema == pa.schema(A		,					metadata=writer._schema.metadata      )
              _check_output(A		,					1      )
def        lowercase_							(A : Union[str, Any]      ):
       if pa.types.is_list(A      ):
              return get_base_dtype(arr_type.value_type      )
       else:
              return arr_type
def        lowercase_							(A : List[Any]		,					A : Union[str, Any]      ):
       if isinstance(lst[0]		,					A      ):
              change_first_primitive_element_in_list(lst[0]		,					A      )
       else:
              snake_case__							:	Dict   	=	value
@pytest.mark.parametrize('optimized_int_type, expected_dtype'		,					[(None, pa.intaa()), (Value('int32'      ), pa.intaa())]      )
@pytest.mark.parametrize('sequence'		,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def        lowercase_							(A : Tuple		,					A : List[str]		,					A : List[Any]      ):
       snake_case__							:	Any   	=	pa.array(TypedSequence(A		,					optimized_int_type=A      )      )
       assert get_base_dtype(arr.type      ) == expected_dtype
@pytest.mark.parametrize(
    'col, expected_dtype'		,					[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]		,					)
@pytest.mark.parametrize('sequence'		,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]      )
def        lowercase_							(A : List[Any]		,					A : Union[str, Any]		,					A : str      ):
       # in range
       snake_case__							:	Dict   	=	pa.array(OptimizedTypedSequence(A		,					col=A      )      )
       assert get_base_dtype(arr.type      ) == expected_dtype

       # not in range
       if col != "other":
              # avoids errors due to in-place modifications
              snake_case__							:	Optional[Any]   	=	copy.deepcopy(A      )
              snake_case__							:	str   	=	np.iinfo(expected_dtype.to_pandas_dtype()      ).max + 1
              change_first_primitive_element_in_list(A		,					A      )
              snake_case__							:	List[str]   	=	pa.array(OptimizedTypedSequence(A		,					col=A      )      )
              assert get_base_dtype(arr.type      ) == pa.intaa()
@pytest.mark.parametrize('raise_exception'		,					[False, True]      )
def        lowercase_							(A : Union[str, Any]		,					A : List[str]      ):
       snake_case__							:	List[str]   	=	str(tmp_path / 'dataset-train.arrow'      )
       try:
              with ArrowWriter(path=A      ) as writer:
                     if raise_exception:
                            raise pa.lib.ArrowInvalid()
                     else:
                            writer.stream.close()
       except pa.lib.ArrowInvalid:
              pass
       finally:
              assert writer.stream.closed
def        lowercase_							(A : int      ):
       snake_case__							:	Any   	=	'mock://dataset-train.arrow'
       with ArrowWriter(path=A		,					storage_options=mockfs.storage_options      ) as writer:
              assert isinstance(writer._fs		,					type(A      )      )
              assert writer._fs.storage_options == mockfs.storage_options
              writer.write({'col_1': 'foo', 'col_2': 1}      )
              writer.write({'col_1': 'bar', 'col_2': 2}      )
              snake_case__	,				snake_case__							:	Union[str, Any]   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       assert mockfs.exists(A      )
def        lowercase_							():
       snake_case__							:	List[str]   	=	pa.BufferOutputStream()
       with ParquetWriter(stream=A      ) as writer:
              writer.write({'col_1': 'foo', 'col_2': 1}      )
              writer.write({'col_1': 'bar', 'col_2': 2}      )
              snake_case__	,				snake_case__							:	Any   	=	writer.finalize()
       assert num_examples == 2
       assert num_bytes > 0
       snake_case__							:	Any   	=	pa.BufferReader(output.getvalue()      )
       snake_case__							:	pa.Table   	=	pq.read_table(A      )
       assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize('embed_local_files'		,					[False, True]      )
def        lowercase_							(A : List[Any]		,					A : Optional[Any]      ):
       import PIL.Image

       snake_case__							:	int   	=	str(tmp_path / 'test_image_rgb.jpg'      )
       PIL.Image.fromarray(np.zeros((5, 5)		,					dtype=np.uinta      )      ).save(A		,					format='png'      )
       snake_case__							:	Tuple   	=	pa.BufferOutputStream()
       with ParquetWriter(
           stream=A		,					features=Features({'image': Image()}      )		,					embed_local_files=A      ) as writer:
              writer.write({'image': image_path}      )
              writer.finalize()
       snake_case__							:	Dict   	=	pa.BufferReader(output.getvalue()      )
       snake_case__							:	pa.Table   	=	pq.read_table(A      )
       snake_case__							:	Optional[Any]   	=	pa_table.to_pydict()
       if embed_local_files:
              assert isinstance(out['image'][0]['path']		,					A      )
              with open(A		,					'rb'      ) as f:
                     assert out["image"][0]["bytes"] == f.read()
       else:
              assert out["image"][0]["path"] == image_path
              assert out["image"][0]["bytes"] is None





def        lowercase_							():
       snake_case__							:	Dict   	=	pa.schema([pa.field('col_1'		,					pa.string()		,					nullable=A      )]      )
       snake_case__							:	Any   	=	pa.BufferOutputStream()
       with ArrowWriter(stream=A      ) as writer:
              writer._build_writer(inferred_schema=A      )
              assert writer._schema == pa.schema([pa.field('col_1'		,					pa.string()      )]      )











from math import factorial
def        lowercase_							(A : int		,					A : int		,					A : float      ):
       if successes > trials:
              raise ValueError('successes must be lower or equal to trials'      )
       if trials < 0 or successes < 0:
              raise ValueError('the function is defined for non-negative integers'      )
       if not isinstance(A		,					A      ) or not isinstance(A		,					A      ):
              raise ValueError('the function is defined for non-negative integers'      )
       if not 0 < prob < 1:
              raise ValueError('prob has to be in range of 1 - 0'      )
       snake_case__							:	List[Any]   	=	(prob**successes) * ((1 - prob) ** (trials - successes))
       # Calculate the binomial coefficient: n! / k!(n-k)!
       snake_case__							:	List[str]   	=	float(factorial(A      )      )
       coefficient /= factorial(A      ) * factorial(trials - successes      )
       return probability * coefficient


if __name__ == "__main__":
 from doctest import testmod

 testmod()
 print("Probability of 2 successes out of 4 trails")
 print("with probability of 0.75 is:", end=" ")
 print(binomial_distribution(2, 4, 0.75))






