'''simple docstring'''

from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
 import torch


_A			:      List[Any]					      =				logging.get_logger(__name__)







class 				_lowercase     (				A_			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       List[str]      	=					['''pixel_values''']
 def __init__(      self   :	Tuple      ,       SCREAMING_SNAKE_CASE__   :	str = True      ,       SCREAMING_SNAKE_CASE__   :	List[str] = None      ,       SCREAMING_SNAKE_CASE__   :	str = PILImageResampling.BILINEAR      ,       SCREAMING_SNAKE_CASE__   :	int = True      ,       SCREAMING_SNAKE_CASE__   :	str = None      ,       SCREAMING_SNAKE_CASE__   :	str = True      ,       SCREAMING_SNAKE_CASE__   :	List[str] = 1 / 2_55      ,       SCREAMING_SNAKE_CASE__   :	Optional[int] = True      ,       SCREAMING_SNAKE_CASE__   :	Tuple = None      ,       SCREAMING_SNAKE_CASE__   :	int = None      ,       **SCREAMING_SNAKE_CASE__   :	Tuple      ,       )  ->    str:
  super().__init__(**SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						size if size is not None else {"""shortest_edge""": 2_56}
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__      ,       default_to_square=SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						crop_size if crop_size is not None else {"""height""": 2_24, """width""": 2_24}
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__      ,       param_name="""crop_size"""							)
  __lowerCAmelCase											=						do_resize
  __lowerCAmelCase											=						size
  __lowerCAmelCase											=						resample
  __lowerCAmelCase											=						do_center_crop
  __lowerCAmelCase											=						crop_size
  __lowerCAmelCase											=						do_rescale
  __lowerCAmelCase											=						rescale_factor
  __lowerCAmelCase											=						do_normalize
  __lowerCAmelCase											=						image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
  __lowerCAmelCase											=						image_std if image_std is not None else IMAGENET_STANDARD_STD
 def a					(      self   :	List[Any]      ,       SCREAMING_SNAKE_CASE__   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	List[str] = PILImageResampling.BICUBIC      ,       SCREAMING_SNAKE_CASE__   :	Tuple = None      ,       **SCREAMING_SNAKE_CASE__   :	Tuple      ,       )  ->    Tuple:
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__      ,       default_to_square=SCREAMING_SNAKE_CASE__							)
  if "shortest_edge" not in size:
   raise ValueError(f"""The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"""							)
  __lowerCAmelCase											=						get_resize_output_image_size(SCREAMING_SNAKE_CASE__      ,       size=size["""shortest_edge"""]      ,       default_to_square=SCREAMING_SNAKE_CASE__							)
  return resize(SCREAMING_SNAKE_CASE__      ,       size=SCREAMING_SNAKE_CASE__      ,       resample=SCREAMING_SNAKE_CASE__      ,       data_format=SCREAMING_SNAKE_CASE__      ,       **SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	Tuple      ,       SCREAMING_SNAKE_CASE__   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Tuple = None      ,       **SCREAMING_SNAKE_CASE__   :	int      ,       )  ->    int:
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__							)
  if "height" not in size or "width" not in size:
   raise ValueError(f"""The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}"""							)
  return center_crop(SCREAMING_SNAKE_CASE__      ,       size=(size["""height"""], size["""width"""])      ,       data_format=SCREAMING_SNAKE_CASE__      ,       **SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	str      ,       SCREAMING_SNAKE_CASE__   :	List[str]      ,       SCREAMING_SNAKE_CASE__   :	Union[str, Any]      ,       SCREAMING_SNAKE_CASE__   :	List[Any] = None      ,       **SCREAMING_SNAKE_CASE__   :	Optional[int]							)  ->    Dict:
  return rescale(SCREAMING_SNAKE_CASE__      ,       scale=SCREAMING_SNAKE_CASE__      ,       data_format=SCREAMING_SNAKE_CASE__      ,       **SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	int      ,       SCREAMING_SNAKE_CASE__   :	Tuple      ,       SCREAMING_SNAKE_CASE__   :	int      ,       SCREAMING_SNAKE_CASE__   :	Tuple = None      ,       **SCREAMING_SNAKE_CASE__   :	Dict      ,       )  ->    Tuple:
  return normalize(SCREAMING_SNAKE_CASE__      ,       mean=SCREAMING_SNAKE_CASE__      ,       std=SCREAMING_SNAKE_CASE__      ,       data_format=SCREAMING_SNAKE_CASE__      ,       **SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	List[Any]      ,       SCREAMING_SNAKE_CASE__   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	Any = None      ,       SCREAMING_SNAKE_CASE__   :	List[Any] = None      ,       SCREAMING_SNAKE_CASE__   :	List[str] = None      ,       SCREAMING_SNAKE_CASE__   :	Optional[int] = None      ,       SCREAMING_SNAKE_CASE__   :	List[Any] = None      ,       SCREAMING_SNAKE_CASE__   :	Any = None      ,       SCREAMING_SNAKE_CASE__   :	Optional[int] = None      ,       SCREAMING_SNAKE_CASE__   :	Optional[int] = None      ,       SCREAMING_SNAKE_CASE__   :	List[Any] = None      ,       SCREAMING_SNAKE_CASE__   :	Dict = None      ,       SCREAMING_SNAKE_CASE__   :	List[Any] = None      ,       SCREAMING_SNAKE_CASE__   :	Union[str, Any] = ChannelDimension.FIRST      ,       **SCREAMING_SNAKE_CASE__   :	Any      ,       )  ->    str:
  __lowerCAmelCase											=						do_resize if do_resize is not None else self.do_resize
  __lowerCAmelCase											=						size if size is not None else self.size
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__      ,       default_to_square=SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						resample if resample is not None else self.resample
  __lowerCAmelCase											=						do_center_crop if do_center_crop is not None else self.do_center_crop
  __lowerCAmelCase											=						crop_size if crop_size is not None else self.crop_size
  __lowerCAmelCase											=						get_size_dict(SCREAMING_SNAKE_CASE__      ,       param_name="""crop_size"""							)
  __lowerCAmelCase											=						do_rescale if do_rescale is not None else self.do_rescale
  __lowerCAmelCase											=						rescale_factor if rescale_factor is not None else self.rescale_factor
  __lowerCAmelCase											=						do_normalize if do_normalize is not None else self.do_normalize
  __lowerCAmelCase											=						image_mean if image_mean is not None else self.image_mean
  __lowerCAmelCase											=						image_std if image_std is not None else self.image_std

  __lowerCAmelCase											=						make_list_of_images(SCREAMING_SNAKE_CASE__							)

  if not valid_images(SCREAMING_SNAKE_CASE__							):
   raise ValueError(
       """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
       """torch.Tensor, tf.Tensor or jax.ndarray."""							)

  if do_resize and size is None:
   raise ValueError("""Size must be specified if do_resize is True."""							)

  if do_center_crop and crop_size is None:
   raise ValueError("""Crop size must be specified if do_center_crop is True."""							)

  if do_rescale and rescale_factor is None:
   raise ValueError("""Rescale factor must be specified if do_rescale is True."""							)

  if do_normalize and (image_mean is None or image_std is None):
   raise ValueError("""Image mean and std must be specified if do_normalize is True."""							)

  # All transformations expect numpy arrays.
  __lowerCAmelCase											=						[to_numpy_array(SCREAMING_SNAKE_CASE__							) for image in images]

  if do_resize:
   __lowerCAmelCase											=						[self.resize(image=SCREAMING_SNAKE_CASE__      ,       size=SCREAMING_SNAKE_CASE__      ,       resample=SCREAMING_SNAKE_CASE__							) for image in images]

  if do_center_crop:
   __lowerCAmelCase											=						[self.center_crop(image=SCREAMING_SNAKE_CASE__      ,       size=SCREAMING_SNAKE_CASE__							) for image in images]

  if do_rescale:
   __lowerCAmelCase											=						[self.rescale(image=SCREAMING_SNAKE_CASE__      ,       scale=SCREAMING_SNAKE_CASE__							) for image in images]

  if do_normalize:
   __lowerCAmelCase											=						[self.normalize(image=SCREAMING_SNAKE_CASE__      ,       mean=SCREAMING_SNAKE_CASE__      ,       std=SCREAMING_SNAKE_CASE__							) for image in images]

  __lowerCAmelCase											=						[to_channel_dimension_format(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							) for image in images]

  __lowerCAmelCase											=						{"""pixel_values""": images}
  return BatchFeature(data=SCREAMING_SNAKE_CASE__      ,       tensor_type=SCREAMING_SNAKE_CASE__							)





 def a					(      self   :	List[str]      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]      ,       SCREAMING_SNAKE_CASE__   :	Any = None							)  ->    Any:
  __lowerCAmelCase											=						outputs.logits

  # Resize logits and compute semantic segmentation maps
  if target_sizes is not None:
   if len(SCREAMING_SNAKE_CASE__							) != len(SCREAMING_SNAKE_CASE__							):
    raise ValueError(
        """Make sure that you pass in as many target sizes as the batch dimension of the logits"""							)

   if is_torch_tensor(SCREAMING_SNAKE_CASE__							):
    __lowerCAmelCase											=						target_sizes.numpy()

   __lowerCAmelCase											=						[]

   for idx in range(len(SCREAMING_SNAKE_CASE__							)							):
    __lowerCAmelCase											=						torch.nn.functional.interpolate(
        logits[idx].unsqueeze(dim=0							)      ,       size=target_sizes[idx]      ,       mode="""bilinear"""      ,       align_corners=SCREAMING_SNAKE_CASE__							)
    __lowerCAmelCase											=						resized_logits[0].argmax(dim=0							)
    semantic_segmentation.append(SCREAMING_SNAKE_CASE__							)
  else:
   __lowerCAmelCase											=						logits.argmax(dim=1							)
   __lowerCAmelCase											=						[semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]							)]

  return semantic_segmentation










import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class       A    (   nn.Module ):


							def __init__(self					):
									super().__init__()
									__lowercase=			nn.Linear(3      ,				4					)
									__lowercase=			nn.BatchNormad(4					)
									__lowercase=			nn.Linear(4      ,				5					)







							def 			_A       (self      ,				lowerCAmelCase					):
									return self.lineara(self.batchnorm(self.lineara(lowerCAmelCase					)					)					)





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				*lowerCAmelCase      ,				**lowerCAmelCase					):
									return (args[0] + 1,) + args[1:], kwargs





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									return output + 1





class       A    (   unittest.TestCase ):


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									self.assertEqual(test_model._hf_hook      ,				lowerCAmelCase					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase      ,				append=lowerCAmelCase					)

									self.assertEqual(isinstance(test_model._hf_hook      ,				lowerCAmelCase					)      ,				lowerCAmelCase					)
									self.assertEqual(len(test_model._hf_hook.hooks					)      ,				2					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(x + 1					)
									__lowercase=			test_model(x + 2					)

									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PreForwardHook()      ,				PreForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PostForwardHook()      ,				PostForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				output + 2      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1					)					)
									self.assertTrue(outputa.requires_grad					)

									__lowercase=			True
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertFalse(outputa.requires_grad					)


							@require_multi_gpu
							def 			_A       (self					):
									__lowercase=			ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=1					)					)

									self.assertEqual(model.lineara.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device(0					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device(1					)					)

									# We can still make a forward pass. The input does not need to be on any particular device
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(1					)					)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(lowerCAmelCase      ,				AlignDevicesHook(io_same_device=lowerCAmelCase					)					)
									__lowercase=			torch.randn(2      ,				3					).to(0					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(0					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(hook_kwargs['execution_device']					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									__lowercase=			{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				offload_buffers=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)







							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()      ,				offload_buffers=lowerCAmelCase      ,				)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)






