






import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      Dict            =						pa.array(TypedSequence([1, 2, 3]   )   )
							self.assertEqual(arr.type				,		pa.intaa()   )



		def 					__UpperCAmelCase			(							self   )	->							Union[str, Any]:






							with self.assertRaises(__lowerCAmelCase   ):
												UpperCAmelCase_    :      List[Any]            =						pa.array(TypedSequence([1, 2, 3]   )				,		type=pa.intaa()   )



		def 					__UpperCAmelCase			(							self   )	->							Any:






							with self.assertRaises(__lowerCAmelCase   ):
												UpperCAmelCase_    :      Union[str, Any]            =						pa.array(TypedSequence([1, 2, 3]				,		try_type=Value('bool'   )				,		type=Value('int64'   )   )   )



		def 					__UpperCAmelCase			(							self   )	->							Optional[int]:
							UpperCAmelCase_    :      Tuple            =						pa.array(TypedSequence([1, 2, 3]				,		type=Value('int32'   )   )   )
							self.assertEqual(arr.type				,		pa.intaa()   )



		def 					__UpperCAmelCase			(							self   )	->							str:






							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
												UpperCAmelCase_    :      Dict            =						pa.array(TypedSequence(['foo', 'bar']				,		type=Value('int64'   )   )   )



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      Any            =						pa.array(TypedSequence([1, 2, 3]				,		try_type=Value('int32'   )   )   )
							self.assertEqual(arr.type				,		pa.intaa()   )



		def 					__UpperCAmelCase			(							self   )	->							Any:
							UpperCAmelCase_    :      Union[str, Any]            =						pa.array(TypedSequence(['foo', 'bar']				,		try_type=Value('int64'   )   )   )
							self.assertEqual(arr.type				,		pa.string()   )



		def 					__UpperCAmelCase			(							self   )	->							Dict:
							UpperCAmelCase_    :      int            =						pa.array(TypedSequence([[[1, 2, 3]]]				,		type=ArrayaD((1, 3)				,		'int64'   )   )   )
							self.assertEqual(arr.type				,		ArrayaDExtensionType((1, 3)				,		'int64'   )   )



		def 					__UpperCAmelCase			(							self   )	->							str:






							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
												UpperCAmelCase_    :      Dict            =						pa.array(TypedSequence(['foo', 'bar']				,		type=ArrayaD((1, 3)				,		'int64'   )   )   )



		def 					__UpperCAmelCase			(							self   )	->							str:
							UpperCAmelCase_    :      Any            =						pa.array(TypedSequence([[[1, 2, 3]]]				,		try_type=ArrayaD((1, 3)				,		'int64'   )   )   )
							self.assertEqual(arr.type				,		ArrayaDExtensionType((1, 3)				,		'int64'   )   )



		def 					__UpperCAmelCase			(							self   )	->							List[Any]:
							UpperCAmelCase_    :      Dict            =						pa.array(TypedSequence(['foo', 'bar']				,		try_type=ArrayaD((1, 3)				,		'int64'   )   )   )
							self.assertEqual(arr.type				,		pa.string()   )



		@require_pil
		def 					__UpperCAmelCase			(							self   )	->							str:
							import PIL.Image

							UpperCAmelCase_    :      Tuple            =						PIL.Image.fromarray(np.arange(1_0				,		dtype=np.uinta   ).reshape(2				,		5   )   )
							with patch(
							    'datasets.arrow_writer.cast_to_python_objects'				,		side_effect=__lowerCAmelCase   ) as mock_cast_to_python_objects:
												UpperCAmelCase_    :      Dict            =						pa.array(TypedSequence([{'path': None, 'bytes': b'image_bytes'}, pil_image]				,		type=Image()   )   )
												UpperCAmelCase_       ,  UpperCAmelCase_    :      List[Any]            =						mock_cast_to_python_objects.call_args_list[-1]
												self.assertIn('optimize_list_casting'				,		__lowerCAmelCase   )
												self.assertFalse(kwargs['optimize_list_casting']   )





def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Union[str, Any]            =						pa.BufferReader(__a  ) if isinstance(__a     ,       pa.Buffer  ) else pa.memory_map(__a  )
					UpperCAmelCase_    :      str            =						pa.ipc.open_stream(__a  )
					UpperCAmelCase_    :      Union[str, Any]            =						f.read_all()
					assert len(pa_table.to_batches()  ) == expected_num_chunks
					assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
					del pa_table




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'     ,       [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						pa.BufferOutputStream()
					UpperCAmelCase_    :      Tuple            =						pa.schema(__a  ) if fields else None
					with ArrowWriter(stream=__a     ,       schema=__a     ,       writer_batch_size=__a  ) as writer:
										writer.write({'col_1': 'foo', 'col_2': 1}  )
										writer.write({'col_1': 'bar', 'col_2': 2}  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      List[Any]            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					if not fields:
										UpperCAmelCase_    :      str            =						{'col_1': pa.string(), 'col_2': pa.intaa()}
					assert writer._schema == pa.schema(__a     ,       metadata=writer._schema.metadata  )
					_check_output(output.getvalue()     ,       expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )




def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						pa.BufferOutputStream()
					UpperCAmelCase_    :      Optional[int]            =						Features({'labels': ClassLabel(names=['neg', 'pos']  )}  )
					with ArrowWriter(stream=__a     ,       features=__a  ) as writer:
										writer.write({'labels': 0}  )
										writer.write({'labels': 1}  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Tuple            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					assert writer._schema == features.arrow_schema
					assert writer._schema.metadata == features.arrow_schema.metadata
					UpperCAmelCase_    :      Dict            =						pa.BufferReader(output.getvalue()  )
					UpperCAmelCase_    :      int            =						pa.ipc.open_stream(__a  )
					UpperCAmelCase_    :      Optional[int]            =						f.read_all()
					UpperCAmelCase_    :      Optional[Any]            =						pa_table.schema
					assert pa_table.num_rows == 2
					assert schema == features.arrow_schema
					assert schema.metadata == features.arrow_schema.metadata
					assert features == Features.from_arrow_schema(__a  )




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 1, 10]  )
def      lowercase__      (     __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						pa.BufferOutputStream()
					with ArrowWriter(
					    stream=__a     ,       writer_batch_size=__a     ,       hash_salt='split_name'     ,       check_duplicates=__a     ,       ) as writer:
										with pytest.raises(__a  ):
															writer.write({'col_1': 'foo', 'col_2': 1}     ,       key=[1, 2]  )
															UpperCAmelCase_       ,  UpperCAmelCase_    :      List[str]            =						writer.finalize()




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 2, 10]  )
def      lowercase__      (     __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Dict            =						pa.BufferOutputStream()
					with ArrowWriter(
					    stream=__a     ,       writer_batch_size=__a     ,       hash_salt='split_name'     ,       check_duplicates=__a     ,       ) as writer:
										with pytest.raises(__a  ):
															writer.write({'col_1': 'foo', 'col_2': 1}     ,       key=10  )
															writer.write({'col_1': 'bar', 'col_2': 2}     ,       key=10  )
															UpperCAmelCase_       ,  UpperCAmelCase_    :      Dict            =						writer.finalize()




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 2, 10]  )
def      lowercase__      (     __snake_case    :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						pa.BufferOutputStream()
					with ArrowWriter(
					    stream=__a     ,       writer_batch_size=__a     ,       hash_salt='split_name'     ,       check_duplicates=__a     ,       ) as writer:
										writer.write({'col_1': 'foo', 'col_2': 1}     ,       key=1  )
										writer.write({'col_1': 'bar', 'col_2': 2}     ,       key=2  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      str            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					_check_output(output.getvalue()     ,       expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'     ,       [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def      lowercase__      (     __snake_case    :							Any     ,       __snake_case    :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						pa.BufferOutputStream()
					UpperCAmelCase_    :      Any            =						pa.schema(__a  ) if fields else None
					with ArrowWriter(stream=__a     ,       schema=__a     ,       writer_batch_size=__a  ) as writer:
										writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )
										writer.write_batch({'col_1': [], 'col_2': []}  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Union[str, Any]            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					if not fields:
										UpperCAmelCase_    :      List[str]            =						{'col_1': pa.string(), 'col_2': pa.intaa()}
					assert writer._schema == pa.schema(__a     ,       metadata=writer._schema.metadata  )
					_check_output(output.getvalue()     ,       expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'     ,       [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						pa.BufferOutputStream()
					UpperCAmelCase_    :      Any            =						pa.schema(__a  ) if fields else None
					with ArrowWriter(stream=__a     ,       schema=__a     ,       writer_batch_size=__a  ) as writer:
										writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      List[str]            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					if not fields:
										UpperCAmelCase_    :      Dict            =						{'col_1': pa.string(), 'col_2': pa.intaa()}
					assert writer._schema == pa.schema(__a     ,       metadata=writer._schema.metadata  )
					_check_output(output.getvalue()     ,       expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )




@pytest.mark.parametrize('writer_batch_size'     ,       [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'     ,       [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							str  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Any            =						pa.BufferOutputStream()
					UpperCAmelCase_    :      Tuple            =						pa.schema(__a  ) if fields else None
					with ArrowWriter(stream=__a     ,       schema=__a     ,       writer_batch_size=__a  ) as writer:
										writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}  )  )
										writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}  )  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Any            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					if not fields:
										UpperCAmelCase_    :      Tuple            =						{'col_1': pa.string(), 'col_2': pa.intaa()}
					assert writer._schema == pa.schema(__a     ,       metadata=writer._schema.metadata  )
					_check_output(output.getvalue()     ,       expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )




def      lowercase__      (     ):


					'''simple docstring'''


					with tempfile.TemporaryDirectory() as tmp_dir:
										UpperCAmelCase_    :      Any            =						{'col_1': pa.string(), 'col_2': pa.intaa()}
										UpperCAmelCase_    :      int            =						os.path.join(__a     ,       'test.arrow'  )
										with ArrowWriter(path=__a     ,       schema=pa.schema(__a  )  ) as writer:
															writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )
															UpperCAmelCase_       ,  UpperCAmelCase_    :      Any            =						writer.finalize()
										assert num_examples == 2
										assert num_bytes > 0
										assert writer._schema == pa.schema(__a     ,       metadata=writer._schema.metadata  )
										_check_output(__a     ,       1  )




def      lowercase__      (     __snake_case    :							Tuple  ):


					'''simple docstring'''


					if pa.types.is_list(__a  ):
										return get_base_dtype(arr_type.value_type  )
					else:
										return arr_type




def      lowercase__      (     __snake_case    :							Any     ,       __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					if isinstance(lst[0]     ,       __a  ):
										change_first_primitive_element_in_list(lst[0]     ,       __a  )
					else:
										UpperCAmelCase_    :      str            =						value




@pytest.mark.parametrize('optimized_int_type, expected_dtype'     ,       [(None, pa.intaa()), (Value('int32'  ), pa.intaa())]  )
@pytest.mark.parametrize('sequence'     ,       [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def      lowercase__      (     __snake_case    :							str     ,       __snake_case    :							Dict     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						pa.array(TypedSequence(__a     ,       optimized_int_type=__a  )  )
					assert get_base_dtype(arr.type  ) == expected_dtype




@pytest.mark.parametrize(
    'col, expected_dtype'     ,       [
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]     ,       )
@pytest.mark.parametrize('sequence'     ,       [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def      lowercase__      (     __snake_case    :							Tuple     ,       __snake_case    :							List[Any]     ,       __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						pa.array(OptimizedTypedSequence(__a     ,       col=__a  )  )
					assert get_base_dtype(arr.type  ) == expected_dtype

					# not in range
					if col != "other":
										# avoids errors due to in-place modifications
										UpperCAmelCase_    :      Any            =						copy.deepcopy(__a  )
										UpperCAmelCase_    :      List[str]            =						np.iinfo(expected_dtype.to_pandas_dtype()  ).max + 1
										change_first_primitive_element_in_list(__a     ,       __a  )
										UpperCAmelCase_    :      Optional[int]            =						pa.array(OptimizedTypedSequence(__a     ,       col=__a  )  )
										assert get_base_dtype(arr.type  ) == pa.intaa()




@pytest.mark.parametrize('raise_exception'     ,       [False, True]  )
def      lowercase__      (     __snake_case    :							Optional[Any]     ,       __snake_case    :							int  ):


					'''simple docstring'''


					UpperCAmelCase_    :      List[str]            =						str(tmp_path / 'dataset-train.arrow'  )
					try:
										with ArrowWriter(path=__a  ) as writer:
															if raise_exception:
																				raise pa.lib.ArrowInvalid()
															else:
																				writer.stream.close()
					except pa.lib.ArrowInvalid:
										pass
					finally:
										assert writer.stream.closed




def      lowercase__      (     __snake_case    :							Tuple  ):


					'''simple docstring'''


					UpperCAmelCase_    :      str            =						'mock://dataset-train.arrow'
					with ArrowWriter(path=__a     ,       storage_options=mockfs.storage_options  ) as writer:
										assert isinstance(writer._fs     ,       type(__a  )  )
										assert writer._fs.storage_options == mockfs.storage_options
										writer.write({'col_1': 'foo', 'col_2': 1}  )
										writer.write({'col_1': 'bar', 'col_2': 2}  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      int            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					assert mockfs.exists(__a  )




def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      int            =						pa.BufferOutputStream()
					with ParquetWriter(stream=__a  ) as writer:
										writer.write({'col_1': 'foo', 'col_2': 1}  )
										writer.write({'col_1': 'bar', 'col_2': 2}  )
										UpperCAmelCase_       ,  UpperCAmelCase_    :      Dict            =						writer.finalize()
					assert num_examples == 2
					assert num_bytes > 0
					UpperCAmelCase_    :      List[str]            =						pa.BufferReader(output.getvalue()  )
					UpperCAmelCase_    :      Tuple            =						pq.read_table(__a  )
					assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}




@require_pil
@pytest.mark.parametrize('embed_local_files'     ,       [False, True]  )
def      lowercase__      (     __snake_case    :							Dict     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					import PIL.Image

					UpperCAmelCase_    :      Optional[int]            =						str(tmp_path / 'test_image_rgb.jpg'  )
					PIL.Image.fromarray(np.zeros((5, 5)     ,       dtype=np.uinta  )  ).save(__a     ,       format='png'  )
					UpperCAmelCase_    :      int            =						pa.BufferOutputStream()
					with ParquetWriter(
					    stream=__a     ,       features=Features({'image': Image()}  )     ,       embed_local_files=__a  ) as writer:
										writer.write({'image': image_path}  )
										writer.finalize()
					UpperCAmelCase_    :      int            =						pa.BufferReader(output.getvalue()  )
					UpperCAmelCase_    :      Tuple            =						pq.read_table(__a  )
					UpperCAmelCase_    :      Any            =						pa_table.to_pydict()
					if embed_local_files:
										assert isinstance(out['image'][0]['path']     ,       __a  )
										with open(__a     ,       'rb'  ) as f:
															assert out["image"][0]["bytes"] == f.read()
					else:
										assert out["image"][0]["path"] == image_path
										assert out["image"][0]["bytes"] is None






def      lowercase__      (     ):


					'''simple docstring'''


					UpperCAmelCase_    :      Any            =						pa.schema([pa.field('col_1'     ,       pa.string()     ,       nullable=__a  )]  )
					UpperCAmelCase_    :      Any            =						pa.BufferOutputStream()
					with ArrowWriter(stream=__a  ) as writer:
										writer._build_writer(inferred_schema=__a  )
										assert writer._schema == pa.schema([pa.field('col_1'     ,       pa.string()  )]  )













import argparse
from argparse import Namespace

import torch
from torch import nn

from transformers import XGLMConfig, XGLMForCausalLM







def    __lowerCamelCase					(							__a   :Dict    )		->				Any:






		"""simple docstring"""

		A__        =   [
		    """decoder.version""",
		    """decoder.output_projection.weight""",
		    """_float_tensor""",
		    """decoder.embed_positions._float_tensor""",
		]
		for k in ignore_keys:
				state_dict.pop(__a    ,     __a    )







def    __lowerCamelCase					(							__a   :str    )		->				Union[str, Any]:






		"""simple docstring"""

		A__      ,			A__        =   emb.weight.shape
		A__        =   nn.Linear(__a    ,     __a    ,     bias=__a    )
		A__        =   emb.weight.data
		return lin_layer







def    __lowerCamelCase					(							__a   :str    )		->				List[str]:






		"""simple docstring"""

		A__        =   torch.load(__a    ,     map_location="""cpu"""    )
		A__        =   Namespace(**checkpoint["""cfg"""]["""model"""]    )
		A__        =   checkpoint["""model"""]
		remove_ignore_keys_(__a    )
		A__        =   state_dict["""decoder.embed_tokens.weight"""].shape[0]

		A__        =   {key.replace("""decoder"""    ,     """model"""    ): val for key, val in state_dict.items()}

		A__        =   XGLMConfig(
		    vocab_size=__a    ,     max_position_embeddings=args.max_target_positions    ,     num_layers=args.decoder_layers    ,     attention_heads=args.decoder_attention_heads    ,     ffn_dim=args.decoder_ffn_embed_dim    ,     d_model=args.decoder_embed_dim    ,     layerdrop=args.decoder_layerdrop    ,     dropout=args.dropout    ,     attention_dropout=args.attention_dropout    ,     activation_dropout=args.activation_dropout    ,     activation_function="""gelu"""    ,     scale_embedding=not args.no_scale_embedding    ,     tie_word_embeddings=args.share_decoder_input_output_embed    ,     )

		A__        =   XGLMForCausalLM(__a    )
		A__        =   model.load_state_dict(__a    ,     strict=__a    )
		print(__a    )
		A__        =   make_linear_from_emb(model.model.embed_tokens    )

		return model


if __name__ == "__main__":
					A    :					int =      argparse.ArgumentParser()
					# Required parameters
					parser.add_argument('''fairseq_path''', type=str, help='''path to a model.pt on local filesystem.''')
					parser.add_argument('''pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
					A    :					str =      parser.parse_args()
					A    :					str =      convert_fairseq_xglm_checkpoint_from_disk(args.fairseq_path)
					model.save_pretrained(args.pytorch_dump_folder_path)



