

import os
import unittest
from tempfile import TemporaryDirectory

import torch
import torch.nn as nn

from accelerate.utils import (
    OffloadedWeightsLoader,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
)

class     a__  (    nn.Module	):
			"""simple docstring"""







			def __init__(			self				)     ->  List[Any]:
							'''simple docstring'''






							super().__init__()
							A__						=	nn.Linear(3					,					4				)
							A__						=	nn.BatchNormad(4				)
							A__						=	nn.Linear(4					,					5				)


			def    UpperCamelCase						(			self					,					lowercase				)     ->  str:
							'''simple docstring'''






							return self.lineara(self.batchnorm(self.lineara(lowercase				)				)				)

class     a__  (    unittest.TestCase	):
			"""simple docstring"""







			def    UpperCamelCase						(			self				)     ->  int:
							'''simple docstring'''






							A__						=	ModelForTest()
							with TemporaryDirectory() as tmp_dir:
											offload_state_dict(lowercase					,					model.state_dict()				)
											A__						=	os.path.join(lowercase					,					"index.json"				)
											self.assertTrue(os.path.isfile(lowercase				)				)
											# TODO: add tests on what is inside the index

											for key in ["linear1.weight", "linear1.bias", "linear2.weight", "linear2.bias"]:
															A__						=	os.path.join(lowercase					,					F'{key}.dat'				)
															self.assertTrue(os.path.isfile(lowercase				)				)
															# TODO: add tests on the fact weights are properly loaded


			def    UpperCamelCase						(			self				)     ->  List[Any]:
							'''simple docstring'''






							A__						=	[torch.floataa, torch.floataa, torch.bfloataa]

							for dtype in dtypes:
											A__						=	torch.randn(2					,					3					,					dtype=lowercase				)
											with TemporaryDirectory() as tmp_dir:
															A__						=	offload_weight(lowercase					,					"weight"					,					lowercase					,					{}				)
															A__						=	os.path.join(lowercase					,					"weight.dat"				)
															self.assertTrue(os.path.isfile(lowercase				)				)
															self.assertDictEqual(lowercase					,					{"weight": {"shape": [2, 3], "dtype": str(lowercase				).split("."				)[1]}}				)

															A__						=	load_offloaded_weight(lowercase					,					index["weight"]				)
															self.assertTrue(torch.equal(lowercase					,					lowercase				)				)


			def    UpperCamelCase						(			self				)     ->  Any:
							'''simple docstring'''






							A__						=	ModelForTest()
							A__						=	model.state_dict()
							A__						=	{k: v for k, v in state_dict.items() if "linear2" not in k}
							A__						=	{k: v for k, v in state_dict.items() if "linear2" in k}

							with TemporaryDirectory() as tmp_dir:
											offload_state_dict(lowercase					,					lowercase				)
											A__						=	OffloadedWeightsLoader(state_dict=lowercase					,					save_folder=lowercase				)

											# Every key is there with the right value
											self.assertEqual(sorted(lowercase				)					,					sorted(state_dict.keys()				)				)
											for key, param in state_dict.items():
															self.assertTrue(torch.allclose(lowercase					,					weight_map[key]				)				)

							A__						=	{k: v for k, v in state_dict.items() if "weight" in k}
							A__						=	{k: v for k, v in state_dict.items() if "weight" not in k}

							with TemporaryDirectory() as tmp_dir:
											offload_state_dict(lowercase					,					lowercase				)
											A__						=	OffloadedWeightsLoader(state_dict=lowercase					,					save_folder=lowercase				)

											# Every key is there with the right value
											self.assertEqual(sorted(lowercase				)					,					sorted(state_dict.keys()				)				)
											for key, param in state_dict.items():
															self.assertTrue(torch.allclose(lowercase					,					weight_map[key]				)				)

							with TemporaryDirectory() as tmp_dir:
											offload_state_dict(lowercase					,					lowercase				)
											# Duplicates are removed
											A__						=	OffloadedWeightsLoader(state_dict=lowercase					,					save_folder=lowercase				)

											# Every key is there with the right value
											self.assertEqual(sorted(lowercase				)					,					sorted(state_dict.keys()				)				)
											for key, param in state_dict.items():
															self.assertTrue(torch.allclose(lowercase					,					weight_map[key]				)				)


			def    UpperCamelCase						(			self				)     ->  List[str]:
							'''simple docstring'''






							A__						=	{"a.1": 0, "a.10": 1, "a.2": 2}
							A__						=	extract_submodules_state_dict(lowercase					,					["a.1", "a.2"]				)
							self.assertDictEqual(lowercase					,					{"a.1": 0, "a.2": 2}				)

							A__						=	{"a.1.a": 0, "a.10.a": 1, "a.2.a": 2}
							A__						=	extract_submodules_state_dict(lowercase					,					["a.1", "a.2"]				)
							self.assertDictEqual(lowercase					,					{"a.1.a": 0, "a.2.a": 2}				)




import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


_UpperCAmelCase		:   Optional[int]										=       5_0000
_UpperCAmelCase		:   Dict										=       5000

_UpperCAmelCase				,						_UpperCAmelCase		:   Optional[int]										=       os.path.split(__file__)
_UpperCAmelCase		:   List[str]										=       os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							for i in range(UpperCamelCase__      ):
														snake_case_							=    dataset[i]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							for i in range(0   ,							len(UpperCamelCase__      )   ,							UpperCamelCase__      ):
														snake_case_							=    dataset[i : i + batch_size]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							with dataset.formatted_as(type=UpperCamelCase__      ):
														for i in range(UpperCamelCase__      ):
																					snake_case_							=    dataset[i]





@get_duration
def 					__lowerCamelCase	( UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__      ):







							'''simple docstring'''






							with dataset.formatted_as(type=UpperCamelCase__      ):
														for i in range(0   ,							UpperCamelCase__   ,							UpperCamelCase__      ):
																					snake_case_							=    dataset[i : i + batch_size]





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							snake_case_							=    {'num examples': SPEED_TEST_N_EXAMPLES}
							snake_case_							=    [
							    (read, {'length': SMALL_TEST}),
							    (read, {'length': SPEED_TEST_N_EXAMPLES}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
							    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'pandas', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'torch', 'length': SMALL_TEST}),
							    (read_formatted, {'type': 'tensorflow', 'length': SMALL_TEST}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
							]

							snake_case_							=    [
							    (read, {'length': SMALL_TEST}),
							    (read, {'length': SPEED_TEST_N_EXAMPLES}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 10}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 100}),
							    (read_batch, {'length': SPEED_TEST_N_EXAMPLES, 'batch_size': 1000}),
							    (read_formatted, {'type': 'numpy', 'length': SMALL_TEST}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 10}),
							    (read_formatted_batch, {'type': 'numpy', 'length': SMALL_TEST, 'batch_size': 1000}),
							]
							with tempfile.TemporaryDirectory() as tmp_dir:
														print('generating dataset'      )
														snake_case_							=    datasets.Features(
														    {'list': datasets.Sequence(datasets.Value('float32'      )      ), 'numbers': datasets.Value('float32'      )}      )
														snake_case_							=    generate_example_dataset(
														    os.path.join(UpperCamelCase__   ,							'dataset.arrow'      )   ,							UpperCamelCase__   ,							num_examples=UpperCamelCase__   ,							seq_shapes={'list': (100,)}   ,							)
														print('first set of iterations'      )
														for func, kwargs in functions:
																					print(func.__name__   ,							str(UpperCamelCase__      )      )
																					snake_case_							=    func(UpperCamelCase__   ,							**UpperCamelCase__      )

														print('shuffling dataset'      )
														snake_case_							=    dataset.shuffle()
														print('Second set of iterations (after shuffling'      )
														for func, kwargs in functions_shuffled:
																					print('shuffled '   ,							func.__name__   ,							str(UpperCamelCase__      )      )
																					snake_case_							=    func(
																					    UpperCamelCase__   ,							**UpperCamelCase__      )

							with open(UpperCamelCase__   ,							'wb'      ) as f:
														f.write(json.dumps(UpperCamelCase__      ).encode('utf-8'      )      )


if __name__ == "__main__":  # useful to run the profiler
					benchmark_iterating()




