


"""simple docstring"""






import sys
from typing import Tuple

import numpy as np
import torch
from PIL import Image
from torch import nn

from transformers.image_utils import PILImageResampling
from utils import img_tensorize





class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : Optional[int] ,       UpperCamelCase    : int ,       UpperCamelCase    : List[Any]=sys.maxsize  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					"""bilinear"""
												__UpperCAmelCase   :						int      							=					max_size
												__UpperCAmelCase   :						List[Any]      							=					short_edge_length






					def __call__(       self    : Any ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					[]
												for img in imgs:
																			__UpperCAmelCase							,__UpperCAmelCase   :						Union[str, Any]      							=					img.shape[:2]
																			# later: provide list and randomly choose index for resize
																			__UpperCAmelCase   :						List[Any]      							=					np.random.randint(self.short_edge_length[0] ,       self.short_edge_length[1] + 1  )
																			if size == 0:
																										return img
																			__UpperCAmelCase   :						Optional[Any]      							=					size * 1.0 / min(UpperCamelCase ,       UpperCamelCase  )
																			if h < w:
																										__UpperCAmelCase							,__UpperCAmelCase   :						str      							=					size, scale * w
																			else:
																										__UpperCAmelCase							,__UpperCAmelCase   :						Any      							=					scale * h, size
																			if max(UpperCamelCase ,       UpperCamelCase  ) > self.max_size:
																										__UpperCAmelCase   :						int      							=					self.max_size * 1.0 / max(UpperCamelCase ,       UpperCamelCase  )
																										__UpperCAmelCase   :						int      							=					newh * scale
																										__UpperCAmelCase   :						List[str]      							=					neww * scale
																			__UpperCAmelCase   :						int      							=					int(neww + 0.5  )
																			__UpperCAmelCase   :						List[Any]      							=					int(newh + 0.5  )

																			if img.dtype == np.uinta:
																										__UpperCAmelCase   :						List[str]      							=					Image.fromarray(UpperCamelCase  )
																										__UpperCAmelCase   :						Optional[int]      							=					pil_image.resize((neww, newh) ,       PILImageResampling.BILINEAR  )
																										__UpperCAmelCase   :						int      							=					np.asarray(UpperCamelCase  )
																			else:
																										__UpperCAmelCase   :						Any      							=					img.permute(2 ,       0 ,       1  ).unsqueeze(0  )  # 3, 0, 1)  # hw(c) -> nchw
																										__UpperCAmelCase   :						int      							=					nn.functional.interpolate(
																										    UpperCamelCase ,       (newh, neww) ,       mode=self.interp_method ,       align_corners=UpperCamelCase  ).squeeze(0  )
																			img_augs.append(UpperCamelCase  )

												return img_augs






class      lowerCamelCase__   :







					"""simple docstring"""


					def __init__(       self    : str ,       UpperCamelCase    : int  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[Any]      							=					ResizeShortestEdge([cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST] ,       cfg.INPUT.MAX_SIZE_TEST  )
												__UpperCAmelCase   :						List[str]      							=					cfg.INPUT.FORMAT
												__UpperCAmelCase   :						str      							=					cfg.SIZE_DIVISIBILITY
												__UpperCAmelCase   :						Optional[int]      							=					cfg.PAD_VALUE
												__UpperCAmelCase   :						List[str]      							=					cfg.INPUT.MAX_SIZE_TEST
												__UpperCAmelCase   :						Optional[int]      							=					cfg.MODEL.DEVICE
												__UpperCAmelCase   :						Tuple      							=					torch.tensor(cfg.MODEL.PIXEL_STD  ).to(self.device  ).view(len(cfg.MODEL.PIXEL_STD  ) ,       1 ,       1  )
												__UpperCAmelCase   :						List[Any]      							=					torch.tensor(cfg.MODEL.PIXEL_MEAN  ).to(self.device  ).view(len(cfg.MODEL.PIXEL_STD  ) ,       1 ,       1  )
												__UpperCAmelCase   :						Optional[int]      							=					lambda UpperCamelCase  : (x - self.pixel_mean) / self.pixel_std






					def lowerCamelCase__							(       self    : Union[str, Any] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					tuple(max(UpperCamelCase  ) for s in zip(*[img.shape for img in images]  )  )
												__UpperCAmelCase   :						int      							=					[im.shape[-2:] for im in images]
												__UpperCAmelCase   :						List[str]      							=					[
												    nn.functional.pad(
												        UpperCamelCase ,       [0, max_size[-1] - size[1], 0, max_size[-2] - size[0]] ,       value=self.pad_value ,       )
												    for size, im in zip(UpperCamelCase ,       UpperCamelCase  )
												]

												return torch.stack(UpperCamelCase  ), torch.tensor(UpperCamelCase  )






					def __call__(       self    : str ,       UpperCamelCase    : Any ,       UpperCamelCase    : List[str]=False  ):
												'''simple docstring'''






												with torch.no_grad():
																			if not isinstance(UpperCamelCase ,       UpperCamelCase  ):
																										__UpperCAmelCase   :						Optional[Any]      							=					[images]
																			if single_image:
																										assert len(UpperCamelCase  ) == 1
																			for i in range(len(UpperCamelCase  )  ):
																										if isinstance(images[i] ,       torch.Tensor  ):
																																	images.insert(UpperCamelCase ,       images.pop(UpperCamelCase  ).to(self.device  ).float()  )
																										elif not isinstance(images[i] ,       torch.Tensor  ):
																																	images.insert(
																																	    UpperCamelCase ,       torch.as_tensor(img_tensorize(images.pop(UpperCamelCase  ) ,       input_format=self.input_format  )  )
																																	    .to(self.device  )
																																	    .float() ,       )
            # resize smallest edge
																			__UpperCAmelCase   :						List[str]      							=					torch.tensor([im.shape[:2] for im in images]  )
																			__UpperCAmelCase   :						List[Any]      							=					self.aug(UpperCamelCase  )
																			# transpose images and convert to torch tensors
																			# images = [torch.as_tensor(i.astype("float32")).permute(2, 0, 1).to(self.device) for i in images]
																			# now normalize before pad to avoid useless arithmetic
																			__UpperCAmelCase   :						Optional[int]      							=					[self.normalizer(UpperCamelCase  ) for x in images]
																			# now pad them to do the following operations
																			__UpperCAmelCase							,__UpperCAmelCase   :						Optional[int]      							=					self.pad(UpperCamelCase  )
																			# Normalize

																			if self.size_divisibility > 0:
																										raise NotImplementedError()
																			# pad
																			__UpperCAmelCase   :						List[Any]      							=					torch.true_divide(UpperCamelCase ,       UpperCamelCase  )
																			if single_image:
																										return images[0], sizes[0], scales_yx[0]
																			else:
																										return images, sizes, scales_yx






def      lowerCamelCase       (     _UpperCamelCase		:      Any       ,			_UpperCamelCase		:      Optional[Any]							)		->       int:

							'''simple docstring'''



							boxes[:, 0::2] *= scale_yx[:, 1]
							boxes[:, 1::2] *= scale_yx[:, 0]
							return boxes
def      lowerCamelCase       (     _UpperCamelCase		:      List[str]       ,			_UpperCamelCase		:      Tuple[int, int]							)		->       str:

							'''simple docstring'''



							assert torch.isfinite(_UpperCamelCase							).all(), "Box tensor contains infinite or NaN!"
							__UpperCAmelCase							,__UpperCAmelCase   :						Optional[Any]      							=					box_size
							tensor[:, 0].clamp_(min=0       ,			max=_UpperCamelCase							)
							tensor[:, 1].clamp_(min=0       ,			max=_UpperCamelCase							)
							tensor[:, 2].clamp_(min=0       ,			max=_UpperCamelCase							)
							tensor[:, 3].clamp_(min=0       ,			max=_UpperCamelCase							)




"""simple docstring"""






import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class      lowerCamelCase__   (     nn.Module				):







					"""simple docstring"""


					def __init__(       self    : List[Any]  ):
												'''simple docstring'''






												super().__init__()
												__UpperCAmelCase   :						Optional[Any]      							=					nn.Linear(3 ,       4  )
												__UpperCAmelCase   :						Optional[int]      							=					nn.BatchNormad(4  )
												__UpperCAmelCase   :						int      							=					nn.Linear(4 ,       5  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												return self.lineara(self.batchnorm(self.lineara(UpperCamelCase  )  )  )






class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : Any ,       *UpperCamelCase    : List[Any] ,       **UpperCamelCase    : Dict  ):
												'''simple docstring'''






												return (args[0] + 1,) + args[1:], kwargs






class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : Dict ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												return output + 1






class      lowerCamelCase__   (     unittest.TestCase				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					ModelForTest()
												__UpperCAmelCase   :						List[str]      							=					ModelHook()

												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												self.assertEqual(test_model._hf_hook ,       UpperCamelCase  )
												self.assertTrue(hasattr(UpperCamelCase ,       """_old_forward"""  )  )

												# Check adding the hook did not change the name or the signature
												self.assertEqual(test_model.forward.__name__ ,       """forward"""  )
												self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,       ["""x"""]  )

												remove_hook_from_module(UpperCamelCase  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_hf_hook"""  )  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_old_forward"""  )  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()
												__UpperCAmelCase   :						Optional[Any]      							=					ModelHook()

												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase ,       append=UpperCamelCase  )

												self.assertEqual(isinstance(test_model._hf_hook ,       UpperCamelCase  ) ,       UpperCamelCase  )
												self.assertEqual(len(test_model._hf_hook.hooks  ) ,       2  )
												self.assertTrue(hasattr(UpperCamelCase ,       """_old_forward"""  )  )

												# Check adding the hook did not change the name or the signature
												self.assertEqual(test_model.forward.__name__ ,       """forward"""  )
												self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,       ["""x"""]  )

												remove_hook_from_module(UpperCamelCase  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_hf_hook"""  )  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_old_forward"""  )  )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						List[str]      							=					test_model(x + 1  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(x + 2  )

												__UpperCAmelCase   :						Optional[Any]      							=					PreForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )  )

												# Attaching a hook to a model when it already has one replaces, does not chain
												__UpperCAmelCase   :						Optional[Any]      							=					PreForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )  )

												# You need to use the sequential hook to chain two or more hooks
												__UpperCAmelCase   :						Optional[Any]      							=					SequentialHook(PreForwardHook() ,       PreForwardHook()  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						List[str]      							=					test_model(UpperCamelCase  )
												assert torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )






					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						int      							=					test_model(UpperCamelCase  )

												__UpperCAmelCase   :						Dict      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Any      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1 ,       atol=1e-5  )  )

												# Attaching a hook to a model when it already has one replaces, does not chain
												__UpperCAmelCase   :						List[str]      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1 ,       atol=1e-5  )  )

												# You need to use the sequential hook to chain two or more hooks
												__UpperCAmelCase   :						Optional[int]      							=					SequentialHook(PostForwardHook() ,       PostForwardHook()  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					test_model(UpperCamelCase  )
												assert torch.allclose(UpperCamelCase ,       output + 2 ,       atol=1e-5  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )

												__UpperCAmelCase   :						Union[str, Any]      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1  )  )
												self.assertTrue(outputa.requires_grad  )

												__UpperCAmelCase   :						List[str]      							=					True
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertFalse(outputa.requires_grad  )






					@require_multi_gpu
					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					ModelForTest()
												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												add_hook_to_module(model.lineara ,       AlignDevicesHook(execution_device=0  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(execution_device=0  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(execution_device=1  )  )

												self.assertEqual(model.lineara.weight.device ,       torch.device(0  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device(0  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device(0  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device(1  )  )

												# We can still make a forward pass. The input does not need to be on any particular device
												__UpperCAmelCase   :						List[str]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       torch.device(1  )  )

												# We can add a general hook to put back output on same device as input.
												add_hook_to_module(UpperCamelCase ,       AlignDevicesHook(io_same_device=UpperCamelCase  )  )
												__UpperCAmelCase   :						int      							=					torch.randn(2 ,       3  ).to(0  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       torch.device(0  )  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Union[str, Any]      							=					{"""execution_device""": 0 if torch.cuda.is_available() else """cpu""", """offload""": True}

												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						Any      							=					torch.device(hook_kwargs["""execution_device"""]  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						Union[str, Any]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[int]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_module(model.lineara  )
												remove_hook_from_module(model.batchnorm  )
												remove_hook_from_module(model.lineara  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												__UpperCAmelCase   :						Union[str, Any]      							=					{
												    """execution_device""": 0 if torch.cuda.is_available() else """cpu""",
												    """offload""": True,
												    """offload_buffers""": True,
												}

												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Dict      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_module(model.lineara  )
												remove_hook_from_module(model.batchnorm  )
												remove_hook_from_module(model.lineara  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )






					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Optional[int]      							=					0 if torch.cuda.is_available() else """cpu"""
												attach_align_device_hook(UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						Any      							=					torch.device(UpperCamelCase  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												attach_align_device_hook(UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       offload_buffers=UpperCamelCase  )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						List[str]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Any      							=					0 if torch.cuda.is_available() else """cpu"""
												attach_align_device_hook(
												    UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       weights_map=model.state_dict()  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						str      							=					torch.device(UpperCamelCase  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						Dict      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[Any]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												attach_align_device_hook(
												    UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       weights_map=model.state_dict() ,       offload_buffers=UpperCamelCase ,       )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						int      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						List[str]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
