


def        _lowerCamelCase(	lowercase__   , lowercase__	)						->	Optional[Any]:

		'''simple docstring'''







		__lowercase=			0
		__lowercase=			len(lowercase__	) - 1

		while left <= right:
				# avoid divided by 0 during interpolation
				if sorted_collection[left] == sorted_collection[right]:
						if sorted_collection[left] == item:
								return left
						else:
								return None

				__lowercase=			left + ((item - sorted_collection[left]) * (right - left)) // (
				    sorted_collection[right] - sorted_collection[left]
				)

				# out of range check
				if point < 0 or point >= len(lowercase__	):
						return None

				__lowercase=			sorted_collection[point]
				if current_item == item:
						return point
				else:
						if point < left:
								__lowercase=			left
								__lowercase=			point
						elif point > right:
								__lowercase=			right
								__lowercase=			point
						else:
								if item < current_item:
										__lowercase=			point - 1
								else:
										__lowercase=			point + 1
		return None







def        _lowerCamelCase(	lowercase__   , lowercase__   , lowercase__   , lowercase__	)						->	Tuple:

		'''simple docstring'''







		if sorted_collection[left] == sorted_collection[right]:
				if sorted_collection[left] == item:
						return left
				else:
						return None

		__lowercase=			left + ((item - sorted_collection[left]) * (right - left)) // (
		    sorted_collection[right] - sorted_collection[left]
		)

		# out of range check
		if point < 0 or point >= len(lowercase__	):
				return None

		if sorted_collection[point] == item:
				return point
		elif point < left:
				return interpolation_search_by_recursion(lowercase__   , lowercase__   , lowercase__   , lowercase__	)
		elif point > right:
				return interpolation_search_by_recursion(lowercase__   , lowercase__   , lowercase__   , lowercase__	)
		else:
				if sorted_collection[point] > item:
						return interpolation_search_by_recursion(
						    lowercase__   , lowercase__   , lowercase__   , point - 1	)
				else:
						return interpolation_search_by_recursion(
						    lowercase__   , lowercase__   , point + 1   , lowercase__	)







def        _lowerCamelCase(	lowercase__	)						->	Union[str, Any]:

		'''simple docstring'''







		if collection != sorted(lowercase__	):
				raise ValueError('Collection must be ascending sorted'	)
		return True


if __name__ == "__main__":
	import sys
	lowerCAmelCase           =		0
	if debug == 1:
		lowerCAmelCase           =		[1_0, 3_0, 4_0, 4_5, 5_0, 6_6, 7_7, 9_3]
		try:
			__assert_sorted(collection)
		except ValueError:
			sys.exit('''Sequence must be ascending sorted to apply interpolation search''')
		lowerCAmelCase           =		6_7

	lowerCAmelCase           =		interpolation_search(collection, target)
	if result is not None:
		print(F'{target} found at positions: {result}')
	else:
		print('''Not found''')










import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class       A    (   nn.Module ):


							def __init__(self					):
									super().__init__()
									__lowercase=			nn.Linear(3      ,				4					)
									__lowercase=			nn.BatchNormad(4					)
									__lowercase=			nn.Linear(4      ,				5					)







							def 			_A       (self      ,				lowerCAmelCase					):
									return self.lineara(self.batchnorm(self.lineara(lowerCAmelCase					)					)					)





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				*lowerCAmelCase      ,				**lowerCAmelCase					):
									return (args[0] + 1,) + args[1:], kwargs





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									return output + 1





class       A    (   unittest.TestCase ):


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									self.assertEqual(test_model._hf_hook      ,				lowerCAmelCase					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase      ,				append=lowerCAmelCase					)

									self.assertEqual(isinstance(test_model._hf_hook      ,				lowerCAmelCase					)      ,				lowerCAmelCase					)
									self.assertEqual(len(test_model._hf_hook.hooks					)      ,				2					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(x + 1					)
									__lowercase=			test_model(x + 2					)

									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PreForwardHook()      ,				PreForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PostForwardHook()      ,				PostForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				output + 2      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1					)					)
									self.assertTrue(outputa.requires_grad					)

									__lowercase=			True
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertFalse(outputa.requires_grad					)


							@require_multi_gpu
							def 			_A       (self					):
									__lowercase=			ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=1					)					)

									self.assertEqual(model.lineara.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device(0					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device(1					)					)

									# We can still make a forward pass. The input does not need to be on any particular device
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(1					)					)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(lowerCAmelCase      ,				AlignDevicesHook(io_same_device=lowerCAmelCase					)					)
									__lowercase=			torch.randn(2      ,				3					).to(0					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(0					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(hook_kwargs['execution_device']					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									__lowercase=			{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				offload_buffers=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)







							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()      ,				offload_buffers=lowerCAmelCase      ,				)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)






