


import math





class       A    :


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									__lowercase=			0.0
									__lowercase=			0.0
									for i in range(len(lowerCAmelCase					)					):
											da += math.pow((sample[i] - weights[0][i])      ,				2					)
											da += math.pow((sample[i] - weights[1][i])      ,				2					)
											return 0 if da > da else 1
									return 0







							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									for i in range(len(lowerCAmelCase					)					):
											weights[j][i] += alpha * (sample[i] - weights[j][i])
									return weights







def        _lowerCamelCase(	)						->	None:

		'''simple docstring'''







		__lowercase=			[[1, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 1]]

		# weight initialization ( n, C )
		__lowercase=			[[0.2, 0.6, 0.5, 0.9], [0.8, 0.4, 0.7, 0.3]]

		# training
		__lowercase=			SelfOrganizingMap()
		__lowercase=			3
		__lowercase=			0.5

		for _ in range(lowercase__	):
				for j in range(len(lowercase__	)	):
						# training sample
						__lowercase=			training_samples[j]

						# Compute the winning vector
						__lowercase=			self_organizing_map.get_winner(lowercase__   , lowercase__	)

						# Update the winning vector
						__lowercase=			self_organizing_map.update(lowercase__   , lowercase__   , lowercase__   , lowercase__	)

    # classify test sample
		__lowercase=			[0, 0, 0, 1]
		__lowercase=			self_organizing_map.get_winner(lowercase__   , lowercase__	)

		# results
		print(F'Clusters that the test sample belongs to : {winner}'	)
		print(F'Weights that have been trained : {weights}'	)


# running the main() function
if __name__ == "__main__":
	main()










import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu





class       A    (   nn.Module ):


							def __init__(self					):
									super().__init__()
									__lowercase=			nn.Linear(3      ,				4					)
									__lowercase=			nn.BatchNormad(4					)
									__lowercase=			nn.Linear(4      ,				5					)







							def 			_A       (self      ,				lowerCAmelCase					):
									return self.lineara(self.batchnorm(self.lineara(lowerCAmelCase					)					)					)





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				*lowerCAmelCase      ,				**lowerCAmelCase					):
									return (args[0] + 1,) + args[1:], kwargs





class       A    (   A_ ):


							def 			_A       (self      ,				lowerCAmelCase      ,				lowerCAmelCase					):
									return output + 1





class       A    (   unittest.TestCase ):


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									self.assertEqual(test_model._hf_hook      ,				lowerCAmelCase					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			ModelHook()

									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase      ,				append=lowerCAmelCase					)

									self.assertEqual(isinstance(test_model._hf_hook      ,				lowerCAmelCase					)      ,				lowerCAmelCase					)
									self.assertEqual(len(test_model._hf_hook.hooks					)      ,				2					)
									self.assertTrue(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)

									# Check adding the hook did not change the name or the signature
									self.assertEqual(test_model.forward.__name__      ,				'forward'					)
									self.assertListEqual(list(inspect.signature(test_model.forward					).parameters					)      ,				['x']					)

									remove_hook_from_module(lowerCAmelCase					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_hf_hook'					)					)
									self.assertFalse(hasattr(lowerCAmelCase      ,				'_old_forward'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(x + 1					)
									__lowercase=			test_model(x + 2					)

									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PreForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PreForwardHook()      ,				PreForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				lowerCAmelCase      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# Attaching a hook to a model when it already has one replaces, does not chain
									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1      ,				atol=1E-5					)					)

									# You need to use the sequential hook to chain two or more hooks
									__lowercase=			SequentialHook(PostForwardHook()      ,				PostForwardHook()					)
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)

									__lowercase=			test_model(lowerCAmelCase					)
									assert torch.allclose(lowerCAmelCase      ,				output + 2      ,				atol=1E-5					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			test_model(lowerCAmelCase					)

									__lowercase=			PostForwardHook()
									add_hook_to_module(lowerCAmelCase      ,				lowerCAmelCase					)
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertTrue(torch.allclose(lowerCAmelCase      ,				output + 1					)					)
									self.assertTrue(outputa.requires_grad					)

									__lowercase=			True
									__lowercase=			test_model(lowerCAmelCase					)
									self.assertFalse(outputa.requires_grad					)


							@require_multi_gpu
							def 			_A       (self					):
									__lowercase=			ModelForTest()
									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(execution_device=0					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(execution_device=1					)					)

									self.assertEqual(model.lineara.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device(0					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device(0					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device(1					)					)

									# We can still make a forward pass. The input does not need to be on any particular device
									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(1					)					)

									# We can add a general hook to put back output on same device as input.
									add_hook_to_module(lowerCAmelCase      ,				AlignDevicesHook(io_same_device=lowerCAmelCase					)					)
									__lowercase=			torch.randn(2      ,				3					).to(0					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				torch.device(0					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(hook_kwargs['execution_device']					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									__lowercase=			{
									    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
									    'offload': True,
									    'offload_buffers': True,
									}

									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.batchnorm      ,				AlignDevicesHook(**lowerCAmelCase					)					)
									add_hook_to_module(model.lineara      ,				AlignDevicesHook(**lowerCAmelCase					)					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_module(model.lineara					)
									remove_hook_from_module(model.batchnorm					)
									remove_hook_from_module(model.lineara					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)


							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				offload_buffers=lowerCAmelCase					)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)







							def 			_A       (self					):
									__lowercase=			ModelForTest()

									# Everything is on CPU
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# This will move each submodule on different devices
									__lowercase=			0 if torch.cuda.is_available() else 'cpu'
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()					)

									# Parameters have been offloaded, so on the meta device
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									# Buffers are not included in the offload by default, so are on the execution device
									__lowercase=			torch.device(lowerCAmelCase					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				lowerCAmelCase					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)

									# Now test with buffers included in the offload
									attach_align_device_hook(
									    lowerCAmelCase      ,				execution_device=lowerCAmelCase      ,				offload=lowerCAmelCase      ,				weights_map=model.state_dict()      ,				offload_buffers=lowerCAmelCase      ,				)

									# Parameters have been offloaded, so on the meta device, buffers included
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('meta'					)					)
									self.assertEqual(model.batchnorm.running_mean.device      ,				torch.device('meta'					)					)

									__lowercase=			torch.randn(2      ,				3					)
									__lowercase=			model(lowerCAmelCase					)
									self.assertEqual(output.device      ,				lowerCAmelCase					)

									# Removing hooks loads back the weights in the model.
									remove_hook_from_submodules(lowerCAmelCase					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.batchnorm.weight.device      ,				torch.device('cpu'					)					)
									self.assertEqual(model.lineara.weight.device      ,				torch.device('cpu'					)					)






