


import math
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import SchedulerMixin, SchedulerOutput





class       A    (   A_					,				A_ ):
							UpperCamelCase_   :							Optional[int]       =1


							@register_to_config
							def __init__(self      ,				lowerCAmelCase = 1_0_0_0      ,				lowerCAmelCase = None					):
									# set `betas`, `alphas`, `timesteps`
									self.set_timesteps(lowerCAmelCase					)

									# standard deviation of the initial noise distribution
									__lowercase=			1.0

									# For now we only support F-PNDM, i.e. the runge-kutta method
									# For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf
									# mainly at formula (9), (12), (13) and the Algorithm 2.
									__lowercase=			4

									# running values
									__lowercase=			[]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase = None					):
									__lowercase=			num_inference_steps
									__lowercase=			torch.linspace(1      ,				0      ,				num_inference_steps + 1					)[:-1]
									__lowercase=			torch.cat([steps, torch.tensor([0.0]					)]					)

									if self.config.trained_betas is not None:
											__lowercase=			torch.tensor(self.config.trained_betas      ,				dtype=torch.floataa					)
									else:
											__lowercase=			torch.sin(steps * math.pi / 2					) ** 2

									__lowercase=			(1.0 - self.betas**2) ** 0.5

									__lowercase=			(torch.atana(self.betas      ,				self.alphas					) / math.pi * 2)[:-1]
									__lowercase=			timesteps.to(lowerCAmelCase					)

									__lowercase=			[]


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase = True      ,				):
									if self.num_inference_steps is None:
											raise ValueError(
											    'Number of inference steps is \'None\', you need to run \'set_timesteps\' after creating the scheduler'					)

									__lowercase=			(self.timesteps == timestep).nonzero().item()
									__lowercase=			timestep_index + 1

									__lowercase=			sample * self.betas[timestep_index] + model_output * self.alphas[timestep_index]
									self.ets.append(lowerCAmelCase					)

									if len(self.ets					) == 1:
											__lowercase=			self.ets[-1]
									elif len(self.ets					) == 2:
											__lowercase=			(3 * self.ets[-1] - self.ets[-2]) / 2
									elif len(self.ets					) == 3:
											__lowercase=			(2_3 * self.ets[-1] - 1_6 * self.ets[-2] + 5 * self.ets[-3]) / 1_2
									else:
											__lowercase=			(1 / 2_4) * (5_5 * self.ets[-1] - 5_9 * self.ets[-2] + 3_7 * self.ets[-3] - 9 * self.ets[-4])

									__lowercase=			self._get_prev_sample(lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					)

									if not return_dict:
											return (prev_sample,)

									return SchedulerOutput(prev_sample=lowerCAmelCase					)


							def 			_A       (self      ,				lowerCAmelCase      ,				*lowerCAmelCase      ,				**lowerCAmelCase					):
									return sample


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase=			self.alphas[timestep_index]
									__lowercase=			self.betas[timestep_index]

									__lowercase=			self.alphas[prev_timestep_index]
									__lowercase=			self.betas[prev_timestep_index]

									__lowercase=			(sample - sigma * ets) / max(lowerCAmelCase      ,				1E-8					)
									__lowercase=			next_alpha * pred + ets * next_sigma

									return prev_sample







							def __len__(self					):
									return self.config.num_train_timesteps










import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class       A    (   nn.Module ):


							def __init__(self					):
									super().__init__()
									__lowercase=			nn.Linear(3      ,				4					)
									__lowercase=			nn.BatchNormad(4					)
									__lowercase=			nn.Linear(4      ,				5					)







							def 			_A       (self      ,				lowerCAmelCase					):
									return self.lineara(self.batchnorm(self.lineara(lowerCAmelCase					)					)					)





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				*lowerCAmelCase      ,				**lowerCAmelCase					):
									return (args[0] + 1,) + args[1:], kwargs





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									return output + 1





class       A    (   unittest.TestCase ):


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									self.assertEqual(test_model._hf_hook      ,				lowerCAmelCase					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase      ,				append=lowerCAmelCase					)

									self.assertEqual(isinstance(test_model._hf_hook      ,				lowerCAmelCase					)      ,				lowerCAmelCase					)
									self.assertEqual(len(test_model._hf_hook.hooks					)      ,				2					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(x + 1					)
									__lowercase=			test_model(x + 2					)

									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PreForwardHook()      ,				PreForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PostForwardHook()      ,				PostForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				output + 2      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1					)					)
									self.assertTrue(outputa.requires_grad					)

									__lowercase=			True
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertFalse(outputa.requires_grad					)


							@require_multi_gpu
							def 			_A       (self					):
									__lowercase=			ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=1					)					)

									self.assertEqual(model.lineara.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device(0					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device(1					)					)

									# We can still make a forward pass. The input does not need to be on any particular device
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(1					)					)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(lowerCAmelCase      ,				AlignDevicesHook(io_same_device=lowerCAmelCase					)					)
									__lowercase=			torch.randn(2      ,				3					).to(0					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(0					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(hook_kwargs['execution_device']					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									__lowercase=			{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				offload_buffers=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)







							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()      ,				offload_buffers=lowerCAmelCase      ,				)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)






