





import gc
import unittest

import torch
from parameterized import parameterized

from diffusers import AutoencoderKL
from diffusers.utils import floats_tensor, load_hf_numpy, require_torch_gpu, slow, torch_all_close, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()





class A_					(			_lowerCamelCase ,      _lowerCamelCase ,      unittest.TestCase   ):
   lowerCAmelCase__        = AutoencoderKL
   lowerCAmelCase__        = """sample"""
   lowerCAmelCase__        = 1e-2




   @property
   def _lowerCAmelCase      (self	:List[Any] )->					List[Any]:
         __A          =				4
         __A          =				3
         __A          =				(32, 32)

         __A          =				floats_tensor((batch_size, num_channels) + sizes ).to(_UpperCamelCase )

         return {"sample": image}




   @property
   def _lowerCAmelCase      (self	:Optional[Any] )->					Dict:
         return (3, 32, 32)




   @property
   def _lowerCAmelCase      (self	:Any )->					Optional[Any]:
         return (3, 32, 32)




   def _lowerCAmelCase      (self	:List[Any] )->					Optional[Any]:
         __A          =				{
             '''block_out_channels''': [32, 64],
             '''in_channels''': 3,
             '''out_channels''': 3,
             '''down_block_types''': ['''DownEncoderBlock2D''', '''DownEncoderBlock2D'''],
             '''up_block_types''': ['''UpDecoderBlock2D''', '''UpDecoderBlock2D'''],
             '''latent_channels''': 4,
         }
         __A          =				self.dummy_input
         return init_dict, inputs_dict




   def _lowerCAmelCase      (self	:Optional[Any] )->					Optional[Any]:
         pass




   def _lowerCAmelCase      (self	:Dict )->					Tuple:
         pass




   @unittest.skipIf(torch_device == '''mps'''    ,   '''Gradient checkpointing skipped on MPS''' )
   def _lowerCAmelCase      (self	:str )->					List[str]:
         # enable deterministic behavior for gradient checkpointing
         __A   ,						__A          =				self.prepare_init_args_and_inputs_for_common()
         __A          =				self.model_class(**_UpperCamelCase )
         model.to(_UpperCamelCase )

         assert not model.is_gradient_checkpointing and model.training

         __A          =				model(**_UpperCamelCase ).sample
         # run the backwards pass on the model. For backwards pass, for simplicity purpose,
         # we won't calculate the loss and rather backprop on out.sum()
         model.zero_grad()

         __A          =				torch.randn_like(_UpperCamelCase )
         __A          =				(out - labels).mean()
         loss.backward()

         # re-instantiate the model now enabling gradient checkpointing
         __A          =				self.model_class(**_UpperCamelCase )
         # clone model
         model_a.load_state_dict(model.state_dict() )
         model_a.to(_UpperCamelCase )
         model_a.enable_gradient_checkpointing()

         assert model_a.is_gradient_checkpointing and model_a.training

         __A          =				model_a(**_UpperCamelCase ).sample
         # run the backwards pass on the model. For backwards pass, for simplicity purpose,
         # we won't calculate the loss and rather backprop on out.sum()
         model_a.zero_grad()
         __A          =				(out_a - labels).mean()
         loss_a.backward()

         # compare the output and parameters gradients
         self.assertTrue((loss - loss_a).abs() < 1e-5 )
         __A          =				dict(model.named_parameters() )
         __A          =				dict(model_a.named_parameters() )
         for name, param in named_params.items():
               self.assertTrue(torch_all_close(param.grad.data    ,   named_params_a[name].grad.data    ,   atol=5e-5 ) )




   def _lowerCAmelCase      (self	:Union[str, Any] )->					Tuple:
         __A   ,						__A          =				AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy'''    ,   output_loading_info=_UpperCamelCase )
         self.assertIsNotNone(_UpperCamelCase )
         self.assertEqual(len(loading_info['''missing_keys'''] )    ,   0 )

         model.to(_UpperCamelCase )
         __A          =				model(**self.dummy_input )

         assert image is not None, "Make sure output is not None"






   def _lowerCAmelCase      (self	:int )->					Dict:
         __A          =				AutoencoderKL.from_pretrained('''fusing/autoencoder-kl-dummy''' )
         __A          =				model.to(_UpperCamelCase )
         model.eval()

         if torch_device == "mps":
               __A          =				torch.manual_seed(0 )
         else:
               __A          =				torch.Generator(device=_UpperCamelCase ).manual_seed(0 )

         __A          =				torch.randn(
             1    ,   model.config.in_channels    ,   model.config.sample_size    ,   model.config.sample_size    ,   generator=torch.manual_seed(0 )    ,   )
         __A          =				image.to(_UpperCamelCase )
         with torch.no_grad():
               __A          =				model(_UpperCamelCase    ,   sample_posterior=_UpperCamelCase    ,   generator=_UpperCamelCase ).sample

         __A          =				output[0, -1, -3:, -3:].flatten().cpu()

         # Since the VAE Gaussian prior's generator is seeded on the appropriate device,
         # the expected output slices are not the same for CPU and GPU.
         if torch_device == "mps":
               __A          =				torch.tensor(
                   [
                       -4.0_078e-01,
                       -3.8_323e-04,
                       -1.2_681e-01,
                       -1.1_462e-01,
                       2.0_095e-01,
                       1.0_893e-01,
                       -8.8_247e-02,
                       -3.0_361e-01,
                       -9.8_644e-03,
                   ] )
         elif torch_device == "cpu":
               __A          =				torch.tensor(
                   [-0.1_3_5_2, 0.0_8_7_8, 0.0_4_1_9, -0.0_8_1_8, -0.1_0_6_9, 0.0_6_8_8, -0.1_4_5_8, -0.4_4_4_6, -0.0_0_2_6] )
         else:
               __A          =				torch.tensor(
                   [-0.2_4_2_1, 0.4_6_4_2, 0.2_5_0_7, -0.0_4_3_8, 0.0_6_8_2, 0.3_1_6_0, -0.2_0_1_8, -0.0_7_2_7, 0.2_4_8_5] )

         self.assertTrue(torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   rtol=1e-2 ) )





@slow
class A_					(			unittest.TestCase   ):




   def _lowerCAmelCase      (self	:Union[str, Any]    ,   _UpperCamelCase	:Any    ,   _UpperCamelCase	:Any )->					Tuple:
         return f"""gaussian_noise_s={seed}_shape={'_'.join([str(_UpperCamelCase ) for s in shape] )}.npy"""




   def _lowerCAmelCase      (self	:str )->					Optional[int]:
         # clean up the VRAM after each test
         super().tearDown()
         gc.collect()
         torch.cuda.empty_cache()




   def _lowerCAmelCase      (self	:Dict    ,   _UpperCamelCase	:Optional[int]=0    ,   _UpperCamelCase	:str=(4, 3, 512, 512)    ,   _UpperCamelCase	:Any=False )->					Union[str, Any]:
         __A          =				torch.floataa if fpaa else torch.floataa
         __A          =				torch.from_numpy(load_hf_numpy(self.get_file_format(_UpperCamelCase    ,   _UpperCamelCase ) ) ).to(_UpperCamelCase ).to(_UpperCamelCase )
         return image




   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:Optional[int]="CompVis/stable-diffusion-v1-4"    ,   _UpperCamelCase	:Optional[Any]=False )->					List[Any]:
         __A          =				'''fp16''' if fpaa else None
         __A          =				torch.floataa if fpaa else torch.floataa

         __A          =				AutoencoderKL.from_pretrained(
             _UpperCamelCase    ,   subfolder='''vae'''    ,   torch_dtype=_UpperCamelCase    ,   revision=_UpperCamelCase    ,   )
         model.to(_UpperCamelCase ).eval()

         return model




   def _lowerCAmelCase      (self	:str    ,   _UpperCamelCase	:List[Any]=0 )->					str:
         if torch_device == "mps":
               return torch.manual_seed(_UpperCamelCase )
         return torch.Generator(device=_UpperCamelCase ).manual_seed(_UpperCamelCase )




   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_3, 0.9_8_7_8, -0.0_4_9_5, -0.0_7_9_0, -0.2_7_0_9, 0.8_3_7_5, -0.2_0_6_0, -0.0_8_2_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_6, 0.1_1_6_8, 0.1_3_3_2, -0.4_8_4_0, -0.2_5_0_8, -0.0_7_9_1, -0.0_4_9_3, -0.4_0_8_9], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ] )
   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:int    ,   _UpperCamelCase	:List[Any]    ,   _UpperCamelCase	:int )->					Optional[int]:
         __A          =				self.get_sd_vae_model()
         __A          =				self.get_sd_image(_UpperCamelCase )
         __A          =				self.get_generator(_UpperCamelCase )

         with torch.no_grad():
               __A          =				model(_UpperCamelCase    ,   generator=_UpperCamelCase    ,   sample_posterior=_UpperCamelCase ).sample

         assert sample.shape == image.shape

         __A          =				sample[-1, -2:, -2:, :2].flatten().float().cpu()
         __A          =				torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice )

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=3e-3 )




   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.0_5_1_3, 0.0_2_8_9, 1.3_7_9_9, 0.2_1_6_6, -0.2_5_7_3, -0.0_8_7_1, 0.5_1_0_3, -0.0_9_9_9]],
           [47, [-0.4_1_2_8, -0.1_3_2_0, -0.3_7_0_4, 0.1_9_6_5, -0.4_1_1_6, -0.2_3_3_2, -0.3_3_4_0, 0.2_2_4_7]],
           # fmt: on
       ] )
   @require_torch_gpu
   def _lowerCAmelCase      (self	:Union[str, Any]    ,   _UpperCamelCase	:Union[str, Any]    ,   _UpperCamelCase	:int )->					Union[str, Any]:
         __A          =				self.get_sd_vae_model(fpaa=_UpperCamelCase )
         __A          =				self.get_sd_image(_UpperCamelCase    ,   fpaa=_UpperCamelCase )
         __A          =				self.get_generator(_UpperCamelCase )

         with torch.no_grad():
               __A          =				model(_UpperCamelCase    ,   generator=_UpperCamelCase    ,   sample_posterior=_UpperCamelCase ).sample

         assert sample.shape == image.shape

         __A          =				sample[-1, -2:, :2, -2:].flatten().float().cpu()
         __A          =				torch.tensor(_UpperCamelCase )

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=1e-2 )




   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.1_6_0_9, 0.9_8_6_6, -0.0_4_8_7, -0.0_7_7_7, -0.2_7_1_6, 0.8_3_6_8, -0.2_0_5_5, -0.0_8_1_4], [-0.2_3_9_5, 0.0_0_9_8, 0.0_1_0_2, -0.0_7_0_9, -0.2_8_4_0, -0.0_2_7_4, -0.0_7_1_8, -0.1_8_2_4]],
           [47, [-0.2_3_7_7, 0.1_1_4_7, 0.1_3_3_3, -0.4_8_4_1, -0.2_5_0_6, -0.0_8_0_5, -0.0_4_9_1, -0.4_0_8_5], [0.0_3_5_0, 0.0_8_4_7, 0.0_4_6_7, 0.0_3_4_4, -0.0_8_4_2, -0.0_5_4_7, -0.0_6_3_3, -0.1_1_3_1]],
           # fmt: on
       ] )
   def _lowerCAmelCase      (self	:Dict    ,   _UpperCamelCase	:Dict    ,   _UpperCamelCase	:Tuple    ,   _UpperCamelCase	:List[str] )->					int:
         __A          =				self.get_sd_vae_model()
         __A          =				self.get_sd_image(_UpperCamelCase )

         with torch.no_grad():
               __A          =				model(_UpperCamelCase ).sample

         assert sample.shape == image.shape

         __A          =				sample[-1, -2:, -2:, :2].flatten().float().cpu()
         __A          =				torch.tensor(expected_slice_mps if torch_device == '''mps''' else expected_slice )

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=3e-3 )




   @parameterized.expand(
       [
           # fmt: off
           [13, [-0.2_0_5_1, -0.1_8_0_3, -0.2_3_1_1, -0.2_1_1_4, -0.3_2_9_2, -0.3_5_7_4, -0.2_9_5_3, -0.3_3_2_3]],
           [37, [-0.2_6_3_2, -0.2_6_2_5, -0.2_1_9_9, -0.2_7_4_1, -0.4_5_3_9, -0.4_9_9_0, -0.3_7_2_0, -0.4_9_2_5]],
           # fmt: on
       ] )
   @require_torch_gpu
   def _lowerCAmelCase      (self	:Any    ,   _UpperCamelCase	:Optional[int]    ,   _UpperCamelCase	:Any )->					Tuple:
         __A          =				self.get_sd_vae_model()
         __A          =				self.get_sd_image(_UpperCamelCase    ,   shape=(3, 4, 64, 64) )

         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         assert list(sample.shape ) == [3, 3, 512, 512]

         __A          =				sample[-1, -2:, :2, -2:].flatten().cpu()
         __A          =				torch.tensor(_UpperCamelCase )

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=1e-3 )




   @parameterized.expand(
       [
           # fmt: off
           [27, [-0.0_3_6_9, 0.0_2_0_7, -0.0_7_7_6, -0.0_6_8_2, -0.1_7_4_7, -0.1_9_3_0, -0.1_4_6_5, -0.2_0_3_9]],
           [16, [-0.1_6_2_8, -0.2_1_3_4, -0.2_7_4_7, -0.2_6_4_2, -0.3_7_7_4, -0.4_4_0_4, -0.3_6_8_7, -0.4_2_7_7]],
           # fmt: on
       ] )
   @require_torch_gpu
   def _lowerCAmelCase      (self	:Optional[Any]    ,   _UpperCamelCase	:Optional[int]    ,   _UpperCamelCase	:Optional[Any] )->					List[Any]:
         __A          =				self.get_sd_vae_model(fpaa=_UpperCamelCase )
         __A          =				self.get_sd_image(_UpperCamelCase    ,   shape=(3, 4, 64, 64)    ,   fpaa=_UpperCamelCase )

         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         assert list(sample.shape ) == [3, 3, 512, 512]

         __A          =				sample[-1, -2:, :2, -2:].flatten().float().cpu()
         __A          =				torch.tensor(_UpperCamelCase )

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=5e-3 )




   @parameterized.expand([(13,), (16,), (27,)] )
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()    ,   reason='''xformers is not required when using PyTorch 2.0.''' )
   def _lowerCAmelCase      (self	:str    ,   _UpperCamelCase	:Optional[int] )->					Dict:
         __A          =				self.get_sd_vae_model(fpaa=_UpperCamelCase )
         __A          =				self.get_sd_image(_UpperCamelCase    ,   shape=(3, 4, 64, 64)    ,   fpaa=_UpperCamelCase )

         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         model.enable_xformers_memory_efficient_attention()
         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         assert list(sample.shape ) == [3, 3, 512, 512]

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=1e-1 )




   @parameterized.expand([(13,), (16,), (37,)] )
   @require_torch_gpu
   @unittest.skipIf(not is_xformers_available()    ,   reason='''xformers is not required when using PyTorch 2.0.''' )
   def _lowerCAmelCase      (self	:Optional[int]    ,   _UpperCamelCase	:List[str] )->					Union[str, Any]:
         __A          =				self.get_sd_vae_model()
         __A          =				self.get_sd_image(_UpperCamelCase    ,   shape=(3, 4, 64, 64) )

         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         model.enable_xformers_memory_efficient_attention()
         with torch.no_grad():
               __A          =				model.decode(_UpperCamelCase ).sample

         assert list(sample.shape ) == [3, 3, 512, 512]

         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=1e-2 )






   @parameterized.expand(
       [
           # fmt: off
           [33, [-0.3_0_0_1, 0.0_9_1_8, -2.6_9_8_4, -3.9_7_2_0, -3.2_0_9_9, -5.0_3_5_3, 1.7_3_3_8, -0.2_0_6_5, 3.4_2_6_7]],
           [47, [-1.5_0_3_0, -4.3_8_7_1, -6.0_3_5_5, -9.1_1_5_7, -1.6_6_6_1, -2.7_8_5_3, 2.1_6_0_7, -5.0_8_2_3, 2.5_6_3_3]],
           # fmt: on
       ] )
   def _lowerCAmelCase      (self	:Optional[Any]    ,   _UpperCamelCase	:List[str]    ,   _UpperCamelCase	:List[str] )->					Dict:
         __A          =				self.get_sd_vae_model()
         __A          =				self.get_sd_image(_UpperCamelCase )
         __A          =				self.get_generator(_UpperCamelCase )

         with torch.no_grad():
               __A          =				model.encode(_UpperCamelCase ).latent_dist
               __A          =				dist.sample(generator=_UpperCamelCase )

         assert list(sample.shape ) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]]

         __A          =				sample[0, -1, -3:, -3:].flatten().cpu()
         __A          =				torch.tensor(_UpperCamelCase )

         __A          =				3e-3 if torch_device != '''mps''' else 1e-2
         assert torch_all_close(_UpperCamelCase    ,   _UpperCamelCase    ,   atol=_UpperCamelCase )












import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
       snake_case__					:							Optional[int]													=     argparse.ArgumentParser(
           description=(
               'Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned'
               ' Distillation'
           )
       )
       parser.add_argument('--model_type', default='bert', choices=['bert'])
       parser.add_argument('--model_name', default='bert-base-uncased', type=str)
       parser.add_argument('--dump_checkpoint', default='serialization_dir/tf_bert-base-uncased_0247911.pth', type=str)
       parser.add_argument('--vocab_transform', action='store_true')
       snake_case__					:							Optional[int]													=     parser.parse_args()

       if args.model_type == "bert":
              snake_case__					:							Dict													=     BertForMaskedLM.from_pretrained(args.model_name)
              snake_case__					:							Union[str, Any]													=     'bert'
       else:
              raise ValueError('args.model_type should be "bert".')

       snake_case__					:							Optional[int]													=     model.state_dict()
       snake_case__					:							List[Any]													=     {}

       for w in ["word_embeddings", "position_embeddings"]:
              snake_case__					:							Tuple													=     state_dict[f'{prefix}.embeddings.{w}.weight']
       for w in ["weight", "bias"]:
              snake_case__					:							Optional[Any]													=     state_dict[f'{prefix}.embeddings.LayerNorm.{w}']

       snake_case__					:							int													=     0
       for teacher_idx in [0, 2, 4, 7, 9, 11]:
              for w in ["weight", "bias"]:
                     snake_case__					:							Union[str, Any]													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'
                     ]
                     snake_case__					:							Dict													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'
                     ]
                     snake_case__					:							int													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'
                     ]

                     snake_case__					:							int													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'
                     ]
                     snake_case__					:							Optional[int]													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'
                     ]

                     snake_case__					:							Optional[Any]													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'
                     ]
                     snake_case__					:							List[str]													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'
                     ]
                     snake_case__					:							int													=     state_dict[
                         f'{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'
                     ]
              std_idx += 1

       snake_case__					:							Optional[int]													=     state_dict['cls.predictions.decoder.weight']
       snake_case__					:							str													=     state_dict['cls.predictions.bias']
       if args.vocab_transform:
              for w in ["weight", "bias"]:
                     snake_case__					:							int													=     state_dict[f'cls.predictions.transform.dense.{w}']
                     snake_case__					:							Optional[int]													=     state_dict[f'cls.predictions.transform.LayerNorm.{w}']

       print(f'N layers selected for distillation: {std_idx}')
       print(f'Number of params transferred for distillation: {len(compressed_sd.keys())}')

       print(f'Save transferred checkpoint to {args.dump_checkpoint}.')
       torch.save(compressed_sd, args.dump_checkpoint)





