


import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowerCAmelCase__      		= 16
lowerCAmelCase__      		= 32
def 							_UpperCAmelCase				(UpperCamelCase__     :  int						):
       return int(x / 2**20						)







class      lowerCAmelCase__       :




     '''simple docstring'''


     def __enter__(						self)			->				str:
            gc.collect()
            torch.cuda.empty_cache()
            torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
            _A      :       Dict				    = torch.cuda.memory_allocated()
            return self


     def __exit__(						self ,						*__lowerCamelCase)			->				Tuple:
            gc.collect()
            torch.cuda.empty_cache()
            _A      :       Any				    = torch.cuda.memory_allocated()
            _A      :       Tuple				    = torch.cuda.max_memory_allocated()
            _A      :       Optional[int]				    = bamb(self.end - self.begin)
            _A      :       Union[str, Any]				    = bamb(self.peak - self.begin)
            # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")




def 							_UpperCAmelCase				(UpperCamelCase__     :  Accelerator				,							UpperCamelCase__     :  int = 16				,							UpperCamelCase__     :  str = "bert-base-cased"				,							UpperCamelCase__     :  int = 320				,							UpperCamelCase__     :  int = 160				,							):
       _A      :       Optional[int]				    = AutoTokenizer.from_pretrained(_UpperCamelCase						)
       _A      :       Tuple				    = load_dataset(
           "glue"				,							"mrpc"				,							split={"train": f"train[:{n_train}]", "validation": f"validation[:{n_val}]"}						)

       def tokenize_function(UpperCamelCase__     :  Union[str, Any]						):
              # max_length=None => use the model max length (it's actually the default)
              _A      :       Optional[Any]				    = tokenizer(examples["sentence1"]				,							examples["sentence2"]				,							truncation=_UpperCamelCase				,							max_length=_UpperCamelCase						)
              return outputs

       # Apply the method we just defined to all the examples in all the splits of the dataset
       _A      :       List[Any]				    = datasets.map(
           _UpperCamelCase				,							batched=_UpperCamelCase				,							remove_columns=["idx", "sentence1", "sentence2"]				,							load_from_cache_file=_UpperCamelCase						)

       # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
       # transformers library
       _A      :       Any				    = tokenized_datasets.rename_column("label"				,							"labels"						)

       def collate_fn(UpperCamelCase__     :  str						):
              # On TPU it's best to pad everything to the same length or training will be very slow.
              if accelerator.distributed_type == DistributedType.TPU:
                     return tokenizer.pad(_UpperCamelCase				,							padding="max_length"				,							max_length=128				,							return_tensors="pt"						)
              return tokenizer.pad(_UpperCamelCase				,							padding="longest"				,							return_tensors="pt"						)

       # Instantiate dataloaders.
       _A      :       Any				    = DataLoader(
           tokenized_datasets["train"]				,							shuffle=_UpperCamelCase				,							collate_fn=_UpperCamelCase				,							batch_size=_UpperCamelCase						)
       _A      :       List[Any]				    = DataLoader(
           tokenized_datasets["validation"]				,							shuffle=_UpperCamelCase				,							collate_fn=_UpperCamelCase				,							batch_size=_UpperCamelCase						)

       return train_dataloader, eval_dataloader
def 							_UpperCAmelCase				(UpperCamelCase__     :  List[Any]				,							UpperCamelCase__     :  str						):
       _A      :       int				    = Accelerator()

       # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
       _A      :       Optional[Any]				    = config["""lr"""]
       _A      :       Any				    = int(config["num_epochs"]						)
       _A      :       int				    = int(config["seed"]						)
       _A      :       Union[str, Any]				    = int(config["batch_size"]						)
       _A      :       str				    = args.model_name_or_path

       set_seed(_UpperCamelCase						)
       _A      :       Dict				    = get_dataloaders(_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							args.n_train				,							args.n_val						)

       # Instantiate the model (we build the model here so that the seed also control new weights initialization)
       _A      :       List[str]				    = AutoModelForSequenceClassification.from_pretrained(_UpperCamelCase				,							return_dict=_UpperCamelCase						)

       # Instantiate optimizer
       _A      :       Optional[Any]				    = (
           AdamW
           if accelerator.state.deepspeed_plugin is None
           or """optimizer""" not in accelerator.state.deepspeed_plugin.deepspeed_config
           else DummyOptim
       )
       _A      :       Optional[int]				    = optimizer_cls(params=model.parameters()				,							lr=_UpperCamelCase						)

       if accelerator.state.deepspeed_plugin is not None:
              _A      :       Optional[int]				    = accelerator.state.deepspeed_plugin.deepspeed_config[
                  """gradient_accumulation_steps"""
              ]
       else:
              _A      :       int				    = 1
       _A      :       Optional[int]				    = (len(_UpperCamelCase						) * num_epochs) // gradient_accumulation_steps

       # Instantiate scheduler
       if (
           accelerator.state.deepspeed_plugin is None
           or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
       ):
              _A      :       List[str]				    = get_linear_schedule_with_warmup(
                  optimizer=_UpperCamelCase				,							num_warmup_steps=0				,							num_training_steps=_UpperCamelCase				,							)
       else:
              _A      :       Union[str, Any]				    = DummyScheduler(_UpperCamelCase				,							total_num_steps=_UpperCamelCase				,							warmup_num_steps=0						)

       # Prepare everything
       # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
       # prepare method.
       _A      :       Tuple				    = accelerator.prepare(
           _UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase				,							_UpperCamelCase						)

       # We need to keep track of how many total steps we have iterated over
       _A      :       Union[str, Any]				    = 0
       # We also need to keep track of the stating epoch so files are named properly
       _A      :       Dict				    = 0

       # Now we train the model
       _A      :       int				    = {}
       for epoch in range(_UpperCamelCase				,							_UpperCamelCase						):
              with TorchTracemalloc() as tracemalloc:
                     model.train()
                     for step, batch in enumerate(_UpperCamelCase						):
                            _A      :       List[str]				    = model(**_UpperCamelCase						)
                            _A      :       Union[str, Any]				    = outputs.loss
                            _A      :       Any				    = loss / gradient_accumulation_steps
                            accelerator.backward(_UpperCamelCase						)
                            if step % gradient_accumulation_steps == 0:
                                   optimizer.step()
                                   lr_scheduler.step()
                                   optimizer.zero_grad()

                            overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
              accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin						)						)						)
              accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used						)						)
              accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked						)						)
              accelerator.print(
                  "Total Peak Memory consumed during the train (max): {}".format(
                      tracemalloc.peaked + bamb(tracemalloc.begin						)						)						)
              _A      :       Tuple				    = tracemalloc.peaked + bamb(tracemalloc.begin						)
              if args.peak_memory_upper_bound is not None:
                     assert (
                         train_total_peak_memory[f"epoch-{epoch}"] <= args.peak_memory_upper_bound
                     ), "Peak memory usage exceeded the upper bound"

       accelerator.wait_for_everyone()
       if accelerator.is_main_process:
              with open(os.path.join(args.output_dir				,							"peak_memory_utilization.json"						)				,							"w"						) as f:
                     json.dump(_UpperCamelCase				,							_UpperCamelCase						)
def 							_UpperCAmelCase				():
       _A      :       Optional[Any]				    = argparse.ArgumentParser(description="Simple example of training script tracking peak GPU memory usage."						)
       parser.add_argument(
           "--model_name_or_path"				,							type=_UpperCamelCase				,							default="bert-base-cased"				,							help="Path to pretrained model or model identifier from huggingface.co/models."				,							required=_UpperCamelCase				,							)
       parser.add_argument(
           "--output_dir"				,							type=_UpperCamelCase				,							default="."				,							help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."				,							)
       parser.add_argument(
           "--peak_memory_upper_bound"				,							type=_UpperCamelCase				,							default=_UpperCamelCase				,							help="The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value."				,							)
       parser.add_argument(
           "--n_train"				,							type=_UpperCamelCase				,							default=320				,							help="Number of training examples to use."				,							)
       parser.add_argument(
           "--n_val"				,							type=_UpperCamelCase				,							default=160				,							help="Number of validation examples to use."				,							)
       parser.add_argument(
           "--num_epochs"				,							type=_UpperCamelCase				,							default=1				,							help="Number of train epochs."				,							)
       _A      :       int				    = parser.parse_args()
       _A      :       int				    = {"""lr""": 2E-5, """num_epochs""": args.num_epochs, """seed""": 42, """batch_size""": 16}
       training_function(_UpperCamelCase				,							_UpperCamelCase						)


if __name__ == "__main__":
    main()




"""simple docstring"""






import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class      lowerCamelCase__   (     nn.Module				):







					"""simple docstring"""


					def __init__(       self    : List[Any]  ):
												'''simple docstring'''






												super().__init__()
												__UpperCAmelCase   :						Optional[Any]      							=					nn.Linear(3 ,       4  )
												__UpperCAmelCase   :						Optional[int]      							=					nn.BatchNormad(4  )
												__UpperCAmelCase   :						int      							=					nn.Linear(4 ,       5  )






					def lowerCamelCase__							(       self    : List[str] ,       UpperCamelCase    : Dict  ):
												'''simple docstring'''






												return self.lineara(self.batchnorm(self.lineara(UpperCamelCase  )  )  )






class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : str ,       UpperCamelCase    : Any ,       *UpperCamelCase    : List[Any] ,       **UpperCamelCase    : Dict  ):
												'''simple docstring'''






												return (args[0] + 1,) + args[1:], kwargs






class      lowerCamelCase__   (     A				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : Any ,       UpperCamelCase    : Dict ,       UpperCamelCase    : List[Any]  ):
												'''simple docstring'''






												return output + 1






class      lowerCamelCase__   (     unittest.TestCase				):







					"""simple docstring"""


					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					ModelForTest()
												__UpperCAmelCase   :						List[str]      							=					ModelHook()

												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												self.assertEqual(test_model._hf_hook ,       UpperCamelCase  )
												self.assertTrue(hasattr(UpperCamelCase ,       """_old_forward"""  )  )

												# Check adding the hook did not change the name or the signature
												self.assertEqual(test_model.forward.__name__ ,       """forward"""  )
												self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,       ["""x"""]  )

												remove_hook_from_module(UpperCamelCase  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_hf_hook"""  )  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_old_forward"""  )  )






					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()
												__UpperCAmelCase   :						Optional[Any]      							=					ModelHook()

												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase ,       append=UpperCamelCase  )

												self.assertEqual(isinstance(test_model._hf_hook ,       UpperCamelCase  ) ,       UpperCamelCase  )
												self.assertEqual(len(test_model._hf_hook.hooks  ) ,       2  )
												self.assertTrue(hasattr(UpperCamelCase ,       """_old_forward"""  )  )

												# Check adding the hook did not change the name or the signature
												self.assertEqual(test_model.forward.__name__ ,       """forward"""  )
												self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,       ["""x"""]  )

												remove_hook_from_module(UpperCamelCase  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_hf_hook"""  )  )
												self.assertFalse(hasattr(UpperCamelCase ,       """_old_forward"""  )  )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Union[str, Any]      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						List[str]      							=					test_model(x + 1  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(x + 2  )

												__UpperCAmelCase   :						Optional[Any]      							=					PreForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )  )

												# Attaching a hook to a model when it already has one replaces, does not chain
												__UpperCAmelCase   :						Optional[Any]      							=					PreForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )  )

												# You need to use the sequential hook to chain two or more hooks
												__UpperCAmelCase   :						Optional[Any]      							=					SequentialHook(PreForwardHook() ,       PreForwardHook()  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						List[str]      							=					test_model(UpperCamelCase  )
												assert torch.allclose(UpperCamelCase ,       UpperCamelCase ,       atol=1e-5  )






					def lowerCamelCase__							(       self    : Dict  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[int]      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						int      							=					test_model(UpperCamelCase  )

												__UpperCAmelCase   :						Dict      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Any      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1 ,       atol=1e-5  )  )

												# Attaching a hook to a model when it already has one replaces, does not chain
												__UpperCAmelCase   :						List[str]      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1 ,       atol=1e-5  )  )

												# You need to use the sequential hook to chain two or more hooks
												__UpperCAmelCase   :						Optional[int]      							=					SequentialHook(PostForwardHook() ,       PostForwardHook()  )
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )

												__UpperCAmelCase   :						int      							=					test_model(UpperCamelCase  )
												assert torch.allclose(UpperCamelCase ,       output + 2 ,       atol=1e-5  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()
												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[Any]      							=					test_model(UpperCamelCase  )

												__UpperCAmelCase   :						Union[str, Any]      							=					PostForwardHook()
												add_hook_to_module(UpperCamelCase ,       UpperCamelCase  )
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertTrue(torch.allclose(UpperCamelCase ,       output + 1  )  )
												self.assertTrue(outputa.requires_grad  )

												__UpperCAmelCase   :						List[str]      							=					True
												__UpperCAmelCase   :						Optional[int]      							=					test_model(UpperCamelCase  )
												self.assertFalse(outputa.requires_grad  )






					@require_multi_gpu
					def lowerCamelCase__							(       self    : Optional[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						str      							=					ModelForTest()
												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												add_hook_to_module(model.lineara ,       AlignDevicesHook(execution_device=0  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(execution_device=0  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(execution_device=1  )  )

												self.assertEqual(model.lineara.weight.device ,       torch.device(0  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device(0  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device(0  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device(1  )  )

												# We can still make a forward pass. The input does not need to be on any particular device
												__UpperCAmelCase   :						List[str]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       torch.device(1  )  )

												# We can add a general hook to put back output on same device as input.
												add_hook_to_module(UpperCamelCase ,       AlignDevicesHook(io_same_device=UpperCamelCase  )  )
												__UpperCAmelCase   :						int      							=					torch.randn(2 ,       3  ).to(0  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       torch.device(0  )  )






					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Union[str, Any]      							=					{"""execution_device""": 0 if torch.cuda.is_available() else """cpu""", """offload""": True}

												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						Any      							=					torch.device(hook_kwargs["""execution_device"""]  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						Union[str, Any]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[int]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_module(model.lineara  )
												remove_hook_from_module(model.batchnorm  )
												remove_hook_from_module(model.lineara  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												__UpperCAmelCase   :						Union[str, Any]      							=					{
												    """execution_device""": 0 if torch.cuda.is_available() else """cpu""",
												    """offload""": True,
												    """offload_buffers""": True,
												}

												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.batchnorm ,       AlignDevicesHook(**UpperCamelCase  )  )
												add_hook_to_module(model.lineara ,       AlignDevicesHook(**UpperCamelCase  )  )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Dict      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_module(model.lineara  )
												remove_hook_from_module(model.batchnorm  )
												remove_hook_from_module(model.lineara  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )






					def lowerCamelCase__							(       self    : Union[str, Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						int      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Optional[int]      							=					0 if torch.cuda.is_available() else """cpu"""
												attach_align_device_hook(UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						Any      							=					torch.device(UpperCamelCase  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						str      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												attach_align_device_hook(UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       offload_buffers=UpperCamelCase  )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						List[str]      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Any      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )






					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					ModelForTest()

												# Everything is on CPU
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# This will move each submodule on different devices
												__UpperCAmelCase   :						Any      							=					0 if torch.cuda.is_available() else """cpu"""
												attach_align_device_hook(
												    UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       weights_map=model.state_dict()  )

												# Parameters have been offloaded, so on the meta device
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												# Buffers are not included in the offload by default, so are on the execution device
												__UpperCAmelCase   :						str      							=					torch.device(UpperCamelCase  )
												self.assertEqual(model.batchnorm.running_mean.device ,       UpperCamelCase  )

												__UpperCAmelCase   :						Dict      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						Optional[Any]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )

												# Now test with buffers included in the offload
												attach_align_device_hook(
												    UpperCamelCase ,       execution_device=UpperCamelCase ,       offload=UpperCamelCase ,       weights_map=model.state_dict() ,       offload_buffers=UpperCamelCase ,       )

												# Parameters have been offloaded, so on the meta device, buffers included
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""meta"""  )  )
												self.assertEqual(model.batchnorm.running_mean.device ,       torch.device("""meta"""  )  )

												__UpperCAmelCase   :						int      							=					torch.randn(2 ,       3  )
												__UpperCAmelCase   :						List[str]      							=					model(UpperCamelCase  )
												self.assertEqual(output.device ,       UpperCamelCase  )

												# Removing hooks loads back the weights in the model.
												remove_hook_from_submodules(UpperCamelCase  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.batchnorm.weight.device ,       torch.device("""cpu"""  )  )
												self.assertEqual(model.lineara.weight.device ,       torch.device("""cpu"""  )  )
