

import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)







def 		_lowercase (     lowercase__						):
      if model.config.model_type == "gpt2":
            return model.transformer.h[0].mlp.c_fc
      return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn







 class   __lowercase      (nn.Module ):
  def __init__(							self      ,							A_      ,							A_					)					->Dict:



        '''simple docstring'''





        super().__init__()
        __lowerCAmelCase					:						str					=					module
        __lowerCAmelCase					:						int					=					nn.Sequential(
            nn.Linear(module.in_features      ,							A_      ,							bias=A_					)      ,							nn.Linear(A_      ,							module.out_features      ,							bias=A_					)      ,							)
        __lowerCAmelCase					:						Optional[int]					=					(2.0 / (5 * min(module.in_features      ,							module.out_features					))) ** 0.5
        nn.init.normal_(self.adapter[0].weight      ,							std=A_					)
        nn.init.zeros_(self.adapter[1].weight					)
        self.adapter.to(module.weight.device					)





  def 	UpperCamelCase__  (							self      ,							A_      ,							*A_      ,							**A_					)					->List[str]:



        '''simple docstring'''





        return self.module(A_      ,							*A_      ,							**A_					) + self.adapter(A_					)







@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class   __lowercase      (unittest.TestCase ):
 # We keep the constants inside the init function and model loading inside setUp function

 # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
 # Therefore here we use only bloom-1b3 to test our module
 _UpperCamelCase			  =  """bigscience/bloom-1b7"""

 # Constant values
 _UpperCamelCase			  =  2.109659552692574

 _UpperCamelCase			  =  """Hello my name is"""
 _UpperCamelCase			  =  set()
 EXPECTED_OUTPUTS.add("""Hello my name is John and I am a professional photographer. I""" )
 EXPECTED_OUTPUTS.add("""Hello my name is John.\nI am a friend of your father.\n""" )
 EXPECTED_OUTPUTS.add("""Hello my name is John Doe, I am a student at the University""" )
 _UpperCamelCase			  =  10
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					AutoTokenizer.from_pretrained(self.model_name					)







class   __lowercase      (_UpperCAmelCase ):
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       super().setUp()

       # Models and tokenizer
       __lowerCAmelCase					:						Optional[int]					=					AutoModelForCausalLM.from_pretrained(
           self.model_name      ,							torch_dtype=torch.floataa      ,							device_map='''auto'''					)
       __lowerCAmelCase					:						Any					=					AutoModelForCausalLM.from_pretrained(self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       del self.model_fpaa
       del self.model_abit

       gc.collect()
       torch.cuda.empty_cache()
 def 	UpperCamelCase__  (							self					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						Any					=					self.model_abit.config

       self.assertTrue(hasattr(A_      ,							'''quantization_config'''					)					)

       __lowerCAmelCase					:						Union[str, Any]					=					config.to_dict()
       __lowerCAmelCase					:						Any					=					config.to_diff_dict()

       __lowerCAmelCase					:						Dict					=					config.to_json_string()
 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       from bitsandbytes.nn import Paramsabit

       __lowerCAmelCase					:						Optional[int]					=					self.model_fpaa.get_memory_footprint()
       __lowerCAmelCase					:						Union[str, Any]					=					self.model_abit.get_memory_footprint()

       self.assertAlmostEqual(mem_fpaa / mem_abit      ,							self.EXPECTED_RELATIVE_DIFFERENCE					)
       __lowerCAmelCase					:						Optional[Any]					=					get_some_linear_layer(self.model_abit					)
       self.assertTrue(linear.weight.__class__ == Paramsabit					)
 def 	UpperCamelCase__  (							self					)					->Tuple:



       '''simple docstring'''





       from transformers import TaPreTrainedModel

       self.model_fpaa.get_memory_footprint()
       self.model_abit.get_memory_footprint()

       for name, module in self.model_abit.named_modules():
             if isinstance(A_      ,							torch.nn.Linear					):
                   if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                         # 4-bit parameters are packed in uint8 variables
                         self.assertTrue(module.weight.dtype == torch.uinta					)
 def 	UpperCamelCase__  (							self					)					->str:



       '''simple docstring'''





       __lowerCAmelCase					:						Tuple					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					)
       __lowerCAmelCase					:						int					=					self.model_abit.generate(input_ids=encoded_input['''input_ids'''].to(0					)      ,							max_new_tokens=10					)

       self.assertIn(self.tokenizer.decode(output_sequences[0]      ,							skip_special_tokens=A_					)      ,							self.EXPECTED_OUTPUTS					)
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[Any]					=					BitsAndBytesConfig()
       __lowerCAmelCase					:						Optional[Any]					=					True

       __lowerCAmelCase					:						List[Any]					=					AutoModelForCausalLM.from_pretrained(
           self.model_name      ,							quantization_config=A_      ,							device_map='''auto'''					)

       __lowerCAmelCase					:						Tuple					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					)
       __lowerCAmelCase					:						Optional[int]					=					model_abit_from_config.generate(
           input_ids=encoded_input['''input_ids'''].to(0					)      ,							max_new_tokens=10					)

       self.assertIn(self.tokenizer.decode(output_sequences[0]      ,							skip_special_tokens=A_					)      ,							self.EXPECTED_OUTPUTS					)
 def 	UpperCamelCase__  (							self					)					->List[str]:



       '''simple docstring'''





       with self.assertRaises(A_					), tempfile.TemporaryDirectory() as tmpdirname:
             self.model_abit.save_pretrained(A_					)
 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       __lowerCAmelCase					:						List[Any]					=					BitsAndBytesConfig()

       with self.assertRaises(A_					):
             __lowerCAmelCase					:						str					=					AutoModelForCausalLM.from_pretrained(
                 self.model_name      ,							quantization_config=A_      ,							load_in_abit=A_      ,							device_map='''auto'''      ,							bnb_abit_quant_type='''nf4'''      ,							)
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       with self.assertRaises(A_					):
             # Tries with `str`
             self.model_abit.to('''cpu'''					)

       with self.assertRaises(A_					):
             # Tries with a `dtype``
             self.model_abit.to(torch.floataa					)

       with self.assertRaises(A_					):
             # Tries with a `device`
             self.model_abit.to(torch.device('''cuda:0'''					)					)

       with self.assertRaises(A_					):
             # Tries with a `device`
             self.model_abit.float()

       with self.assertRaises(A_					):
             # Tries with a `device`
             self.model_abit.half()

       # Test if we did not break anything
       __lowerCAmelCase					:						Dict					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					)

       __lowerCAmelCase					:						Optional[Any]					=					self.model_fpaa.to(torch.floataa					)
       __lowerCAmelCase					:						Any					=					self.model_fpaa.generate(input_ids=encoded_input['''input_ids'''].to(0					)      ,							max_new_tokens=10					)

       # Check this does not throw an error
       __lowerCAmelCase					:						Optional[Any]					=					self.model_fpaa.to('''cpu'''					)

       # Check this does not throw an error
       __lowerCAmelCase					:						List[str]					=					self.model_fpaa.half()

       # Check this does not throw an error
       __lowerCAmelCase					:						Tuple					=					self.model_fpaa.float()





 def 	UpperCamelCase__  (							self					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					AutoModelForSeqaSeqLM.from_pretrained('''t5-small'''      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa					)







@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class   __lowercase      (unittest.TestCase ):
 @classmethod
 def 	UpperCamelCase__  (							cls					)					->Any:



       '''simple docstring'''





       __lowerCAmelCase					:						str					=					'''t5-small'''
       __lowerCAmelCase					:						Optional[int]					=					'''google/flan-t5-small'''  # flan-t5 uses dense-act instead of dense-relu-dense
       __lowerCAmelCase					:						List[str]					=					AutoTokenizer.from_pretrained(cls.model_name					)
       __lowerCAmelCase					:						List[Any]					=					'''Translate in German: Hello, my dog is cute'''
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       gc.collect()
       torch.cuda.empty_cache()
 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       from transformers import TaForConditionalGeneration

       __lowerCAmelCase					:						List[str]					=					TaForConditionalGeneration._keep_in_fpaa_modules
       __lowerCAmelCase					:						Tuple					=					None

       # test with `t5-small`
       __lowerCAmelCase					:						Dict					=					TaForConditionalGeneration.from_pretrained(self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       __lowerCAmelCase					:						Tuple					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					).to(0					)
       __lowerCAmelCase					:						str					=					model.generate(**A_					)

       # test with `flan-t5-small`
       __lowerCAmelCase					:						Optional[int]					=					TaForConditionalGeneration.from_pretrained(
           self.dense_act_model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       __lowerCAmelCase					:						Union[str, Any]					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					).to(0					)
       __lowerCAmelCase					:						Tuple					=					model.generate(**A_					)
       __lowerCAmelCase					:						List[str]					=					modules





 def 	UpperCamelCase__  (							self					)					->List[Any]:



       '''simple docstring'''





       import bitsandbytes as bnb

       from transformers import TaForConditionalGeneration

       # test with `t5-small`
       __lowerCAmelCase					:						Optional[Any]					=					TaForConditionalGeneration.from_pretrained(self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)

       # there was a bug with decoders - this test checks that it is fixed
       self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q      ,							bnb.nn.Linearabit					)					)

       __lowerCAmelCase					:						Dict					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					).to(0					)
       __lowerCAmelCase					:						List[str]					=					model.generate(**A_					)

       # test with `flan-t5-small`
       __lowerCAmelCase					:						Dict					=					TaForConditionalGeneration.from_pretrained(
           self.dense_act_model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       __lowerCAmelCase					:						Dict					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					).to(0					)
       __lowerCAmelCase					:						Tuple					=					model.generate(**A_					)







class   __lowercase      (_UpperCAmelCase ):
 def 	UpperCamelCase__  (							self					)					->Union[str, Any]:



       '''simple docstring'''





       super().setUp()
       # model_name
       __lowerCAmelCase					:						str					=					'''bigscience/bloom-560m'''
       __lowerCAmelCase					:						Any					=					'''t5-small'''

       # Different types of model

       __lowerCAmelCase					:						int					=					AutoModel.from_pretrained(self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       # Sequence classification model
       __lowerCAmelCase					:						int					=					AutoModelForSequenceClassification.from_pretrained(
           self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       # CausalLM model
       __lowerCAmelCase					:						Any					=					AutoModelForCausalLM.from_pretrained(self.model_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
       # Seq2seq model
       __lowerCAmelCase					:						Optional[int]					=					AutoModelForSeqaSeqLM.from_pretrained(
           self.seq_to_seq_name      ,							load_in_abit=A_      ,							device_map='''auto'''					)
 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       del self.base_model
       del self.sequence_model
       del self.model_abit
       del self.seq_to_seq_model

       gc.collect()
       torch.cuda.empty_cache()





 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       from bitsandbytes.nn import Paramsabit

       self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit					)

       # Other heads should be nn.Parameter
       self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter					)
       self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter					)
       self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter					)







class   __lowercase      (_UpperCAmelCase ):
 def 	UpperCamelCase__  (							self					)					->Dict:



       '''simple docstring'''





       super().setUp()
 def 	UpperCamelCase__  (							self					)					->Optional[int]:



       '''simple docstring'''





       del self.pipe

       gc.collect()
       torch.cuda.empty_cache()





 def 	UpperCamelCase__  (							self					)					->List[str]:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					pipeline(
           '''text-generation'''      ,							model=self.model_name      ,							model_kwargs={'''device_map''': '''auto''', '''load_in_4bit''': True, '''torch_dtype''': torch.floataa}      ,							max_new_tokens=self.MAX_NEW_TOKENS      ,							)

       # Real second forward pass
       __lowerCAmelCase					:						Any					=					self.pipe(self.input_text					)
       self.assertIn(pipeline_output[0]['''generated_text''']      ,							self.EXPECTED_OUTPUTS					)







@require_torch_multi_gpu
class   __lowercase      (_UpperCAmelCase ):
 def 	UpperCamelCase__  (							self					)					->List[str]:



       '''simple docstring'''





       super().setUp()





 def 	UpperCamelCase__  (							self					)					->Tuple:



       '''simple docstring'''





       __lowerCAmelCase					:						Optional[int]					=					AutoModelForCausalLM.from_pretrained(
           self.model_name      ,							load_in_abit=A_      ,							device_map='''balanced'''					)

       # Check correct device map
       self.assertEqual(set(model_parallel.hf_device_map.values()					)      ,							{0, 1}					)

       # Check that inference pass works on the model
       __lowerCAmelCase					:						List[Any]					=					self.tokenizer(self.input_text      ,							return_tensors='''pt'''					)

       # Second real batch
       __lowerCAmelCase					:						Any					=					model_parallel.generate(input_ids=encoded_input['''input_ids'''].to(0					)      ,							max_new_tokens=10					)
       self.assertIn(self.tokenizer.decode(output_parallel[0]      ,							skip_special_tokens=A_					)      ,							self.EXPECTED_OUTPUTS					)







class   __lowercase      (_UpperCAmelCase ):
 def 	UpperCamelCase__  (							self					)					->int:



       '''simple docstring'''





       __lowerCAmelCase					:						int					=					'''facebook/opt-350m'''
       super().setUp()





 def 	UpperCamelCase__  (							self					)					->Optional[Any]:



       '''simple docstring'''





       if version.parse(importlib.metadata.version('''bitsandbytes'''					)					) < version.parse('''0.37.0'''					):
             return

       # Step 1: freeze all parameters
       __lowerCAmelCase					:						str					=					AutoModelForCausalLM.from_pretrained(self.model_name      ,							load_in_abit=A_					)

       self.assertEqual(set(model.hf_device_map.values()					)      ,							{torch.cuda.current_device()}					)

       for param in model.parameters():
             __lowerCAmelCase					:						List[Any]					=					False  # freeze the model - train adapters later
             if param.ndim == 1:
                   # cast the small parameters (e.g. layernorm) to fp32 for stability
                   __lowerCAmelCase					:						Dict					=					param.data.to(torch.floataa					)

        # Step 2: add adapters
       for _, module in model.named_modules():
             if "OPTAttention" in repr(type(A_					)					):
                   __lowerCAmelCase					:						List[Any]					=					LoRALayer(module.q_proj      ,							rank=16					)
                   __lowerCAmelCase					:						List[Any]					=					LoRALayer(module.k_proj      ,							rank=16					)
                   __lowerCAmelCase					:						int					=					LoRALayer(module.v_proj      ,							rank=16					)

        # Step 3: dummy batch
       __lowerCAmelCase					:						str					=					self.tokenizer('''Test batch '''      ,							return_tensors='''pt'''					).to(0					)

       # Step 4: Check if the gradient is not None
       with torch.cuda.amp.autocast():
             __lowerCAmelCase					:						Optional[Any]					=					model.forward(**A_					)
             out.logits.norm().backward()

       for module in model.modules():
             if isinstance(A_      ,							A_					):
                   self.assertTrue(module.adapter[1].weight.grad is not None					)
                   self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0					)
             elif isinstance(A_      ,							nn.Embedding					):
                   self.assertTrue(module.weight.grad is None					)







class   __lowercase      (_UpperCAmelCase ):
 _UpperCamelCase			  =  """gpt2-xl"""
 _UpperCamelCase			  =  3.3191854854152187





from typing import Dict

import numpy as np
import torch

from . import residue_constants as rc
from .tensor_utils import tensor_tree_map, tree_map







def 		_lowercase (     lowercase__						):
      __lowerCAmelCase					:						str					=					[]
      __lowerCAmelCase					:						List[Any]					=					[]
      __lowerCAmelCase					:						str					=					[]

      for rt in rc.restypes:
            __lowerCAmelCase					:						List[Any]					=					rc.restype_name_to_atomaa_names[rc.restype_atoa[rt]]
            restype_atomaa_to_atomaa_list.append([(rc.atom_order[name] if name else 0) for name in atom_names]						)
            __lowerCAmelCase					:						List[str]					=					{name: i for i, name in enumerate(lowercase__						)}
            restype_atomaa_to_atomaa_list.append(
                [(atom_name_to_idxaa[name] if name in atom_name_to_idxaa else 0) for name in rc.atom_types]						)

            restype_atomaa_mask_list.append([(1.0 if name else 0.0) for name in atom_names]						)

      # Add dummy mapping for restype 'UNK'
      restype_atomaa_to_atomaa_list.append([0] * 1_4						)
      restype_atomaa_to_atomaa_list.append([0] * 3_7						)
      restype_atomaa_mask_list.append([0.0] * 1_4						)

      __lowerCAmelCase					:						List[Any]					=					torch.tensor(
          lowercase__   ,	dtype=torch.intaa   ,	device=protein['''aatype'''].device   ,	)
      __lowerCAmelCase					:						Optional[Any]					=					torch.tensor(
          lowercase__   ,	dtype=torch.intaa   ,	device=protein['''aatype'''].device   ,	)
      __lowerCAmelCase					:						Tuple					=					torch.tensor(
          lowercase__   ,	dtype=torch.floataa   ,	device=protein['''aatype'''].device   ,	)
      __lowerCAmelCase					:						List[Any]					=					protein['''aatype'''].to(torch.long						)

      # create the mapping for (residx, atom14) --> atom37, i.e. an array
      # with shape (num_res, 14) containing the atom37 indices for this protein
      __lowerCAmelCase					:						Any					=					restype_atomaa_to_atomaa[protein_aatype]
      __lowerCAmelCase					:						Union[str, Any]					=					restype_atomaa_mask[protein_aatype]

      __lowerCAmelCase					:						int					=					residx_atomaa_mask
      __lowerCAmelCase					:						List[str]					=					residx_atomaa_to_atomaa.long()

      # create the gather indices for mapping back
      __lowerCAmelCase					:						int					=					restype_atomaa_to_atomaa[protein_aatype]
      __lowerCAmelCase					:						Union[str, Any]					=					residx_atomaa_to_atomaa.long()

      # create the corresponding mask
      __lowerCAmelCase					:						str					=					torch.zeros([2_1, 3_7]   ,	dtype=torch.floataa   ,	device=protein['''aatype'''].device						)
      for restype, restype_letter in enumerate(rc.restypes						):
            __lowerCAmelCase					:						Optional[int]					=					rc.restype_atoa[restype_letter]
            __lowerCAmelCase					:						Optional[Any]					=					rc.residue_atoms[restype_name]
            for atom_name in atom_names:
                  __lowerCAmelCase					:						str					=					rc.atom_order[atom_name]
                  __lowerCAmelCase					:						List[Any]					=					1

      __lowerCAmelCase					:						Union[str, Any]					=					restype_atomaa_mask[protein_aatype]
      __lowerCAmelCase					:						Any					=					residx_atomaa_mask

      return protein







def 		_lowercase (     lowercase__						):
      __lowerCAmelCase					:						Dict					=					tree_map(lambda lowercase__						: torch.tensor(lowercase__   ,	device=batch['''aatype'''].device						)   ,	lowercase__   ,	np.ndarray						)
      __lowerCAmelCase					:						Tuple					=					tensor_tree_map(lambda lowercase__						: np.array(lowercase__						)   ,	make_atomaa_masks(lowercase__						)						)
      return out


