codingutf8 2022 the huggingface team inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a clone of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license wraps a linear layer with loralike adapter used for testing purposes only def initself module nn module rank int super init self module module self adapter nn sequential nn linearmodule infeatures rank biasfalse nn linearrank module outfeatures biasfalse smallstd 2 0 5 minmodule infeatures module outfeatures 0 5 nn init normalself adapter0 weight stdsmallstd nn init zerosself adapter1 weight self adapter tomodule weight device def forwardself input args kwargs return self moduleinput args kwargs self adapterinput requirebitsandbytes requireaccelerate requiretorch requiretorchgpu slow class base4bittestunittest testcase we keep the constants inside the init function and model loading inside setup function we need to test on relatively large models aka 1b parameters otherwise the quantiztion may not work as expected therefore here we use only bloom1b3 to test our module modelname bigsciencebloom1b7 constant values expectedrelativedifference 2 109659552692574 this was obtained on a rtx titan so the number might slightly change inputtext hello my name is expectedoutputs set expectedoutputs addhello my name is john and i am a professional photographer i expectedoutputs addhello my name is john ni am a friend of your father n expectedoutputs addhello my name is john doe i am a student at the university maxnewtokens 10 def setupself models and tokenizer self tokenizer autotokenizer frompretrainedself modelname class bnb4bittestbase4bittest def setupself super setup models and tokenizer self modelfp16 automodelforcausallm frompretrained self modelname torchdtypetorch float16 devicemapauto self model4bit automodelforcausallm frompretrainedself modelname loadin4bittrue devicemapauto def teardownself r teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch orgthowcanwereleasegpumemorycache1453027 numparams4bit self model4bit numparameters numparamsfp16 self modelfp16 numparameters self assertequalnumparams4bit numparamsfp16 def testquantizationconfigjsonserializationself r a simple test to check if the quantization config is correctly serialized and deserialized from bitsandbytes nn import params4bit memfp16 self modelfp16 getmemoryfootprint mem4bit self model4bit getmemoryfootprint self assertalmostequalmemfp16 mem4bit self expectedrelativedifference linear getsomelinearlayerself model4bit self asserttruelinear weight class params4bit def testoriginaldtypeself r a simple test to check if the model succesfully stores the original dtype from transformers import t5pretrainedmodel self modelfp16 getmemoryfootprint self model4bit getmemoryfootprint for name module in self model4bit namedmodules if isinstancemodule torch nn linear if name not in lmhead t5pretrainedmodel keepinfp32modules 4bit parameters are packed in uint8 variables self asserttruemodule weight dtype torch uint8 def testrwkv4bitself r a simple test to check if 4bit rwkv inference works as expected encodedinput self tokenizerself inputtext returntensorspt outputsequences self model4bit generateinputidsencodedinputinputids to0 maxnewtokens10 self assertinself tokenizer decodeoutputsequences0 skipspecialtokenstrue self expectedoutputs def testgeneratequalityconfigself r test that loading the model with the config is equivalent with self assertraisesnotimplementederror tempfile temporarydirectory as tmpdirname self model4bit savepretrainedtmpdirname def testraiseifconfigandloadin4bitself r test that loading the model with the config and loadin4bit raises an error with self assertraisesvalueerror tries with str self model4bit tocpu with self assertraisesvalueerror tries with a dtype self model4bit totorch float16 with self assertraisesvalueerror tries with a device self model4bit totorch devicecuda 0 with self assertraisesvalueerror tries with a device self model4bit float with self assertraisesvalueerror tries with a device self model4bit half test if we did not break anything encodedinput self tokenizerself inputtext returntensorspt self modelfp16 self modelfp16 totorch float32 self modelfp16 generateinputidsencodedinputinputids to0 maxnewtokens10 check this does not throw an error self modelfp16 tocpu check this does not throw an error self modelfp16 half check this does not throw an error self modelfp16 float def testfp324bitconversionself r test whether it is possible to mix both 4bit and fp32 weights when using keepinfp32modules correctly gc collect torch cuda emptycache def testinferencewithoutkeepinfp32self r test whether it is possible to mix both 4bit and fp32 weights when using keepinfp32modules correctly flant5small uses t5densegatedactdense whereas t5small uses t5densereludense we need to test both cases test with t5small test with flant5small import bitsandbytes as bnb from transformers import t5forconditionalgeneration test with t5small model t5forconditionalgeneration frompretrainedself modelname loadin4bittrue devicemapauto there was a bug with decoders this test checks that it is fixed self asserttrueisinstancemodel decoder block0 layer0 selfattention q bnb nn linear4bit encodedinput self tokenizerself inputtext returntensorspt to0 model generateencodedinput test with flant5small model t5forconditionalgeneration frompretrained self denseactmodelname loadin4bittrue devicemapauto encodedinput self tokenizerself inputtext returntensorspt to0 model generateencodedinput class classes4bitmodeltestbase4bittest def setupself super setup modelname self modelname bigsciencebloom560m self seqtoseqname t5small different types of model self basemodel automodel frompretrainedself modelname loadin4bittrue devicemapauto sequence classification model self sequencemodel automodelforsequenceclassification frompretrained self modelname loadin4bittrue devicemapauto causallm model self model4bit automodelforcausallm frompretrainedself modelname loadin4bittrue devicemapauto seq2seq model self seqtoseqmodel automodelforseq2seqlm frompretrained self seqtoseqname loadin4bittrue devicemapauto def teardownself r teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch orgthowcanwereleasegpumemorycache1453027 from bitsandbytes nn import params4bit self asserttrueself basemodel h1 mlp dense4htoh weight class params4bit other heads should be nn parameter self asserttrueself model4bit lmhead weight class torch nn parameter self asserttrueself sequencemodel score weight class torch nn parameter self asserttrueself seqtoseqmodel lmhead weight class torch nn parameter class pipeline4bittestbase4bittest def setupself super setup def teardownself r teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch orgthowcanwereleasegpumemorycache1453027 self clearcudacache self pipe pipeline textgeneration modelself modelname modelkwargsdevicemap auto loadin4bit true torchdtype torch float16 maxnewtokensself maxnewtokens real second forward pass pipelineoutput self pipeself inputtext self assertinpipelineoutput0generatedtext self expectedoutputs requiretorchmultigpu class bnb4bittestmultigpubase4bittest def setupself super setup def testmultigpuloadingself r this tests that the model has been loaded and can be used correctly on a multigpu setup let s just try to load a model on 2 gpus and see if it works the model we test has 2gb of total 3gb should suffice check correct device map check that inference pass works on the model second real batch step 1 freeze all parameters cast the small parameters e g layernorm to fp32 for stability step 2 add adapters step 3 dummy batch step 4 check if the gradient is not none coding utf 8 2022 the huggingface team inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a clone of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license wraps a linear layer with lora like adapter used for testing purposes only we keep the constants inside the init function and model loading inside setup function we need to test on relatively large models aka 1b parameters otherwise the quantiztion may not work as expected therefore here we use only bloom 1b3 to test our module constant values this was obtained on a rtx titan so the number might slightly change models and tokenizer models and tokenizer teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch org t how can we release gpu memory cache 14530 27 test if the number of returned parameters is correct see https github com huggingface transformers issues 25978 a simple test to check if the quantization config is correctly serialized and deserialized a simple test to check if the model conversion has been done correctly by checking on the memory footprint of the converted model and the class type of the linear layers of the converted models a simple test to check if the model succesfully stores the original dtype a simple test to check if the model conversion has been done correctly by checking on the memory footprint of the converted model and the class type of the linear layers of the converted models 4 bit parameters are packed in uint8 variables a simple test to check if 4 bit rwkv inference works as expected test the generation quality of the quantized model and see that we are matching the expected output given that we are operating on small numbers the testing model is relatively small we might not get the same output across gpus so we ll generate few tokens 5 10 and check their output test that loading the model with the config is equivalent test whether trying to save a model after converting it in 8 bit will throw a warning test that loading the model with the config and load_in_4bit raises an error test whether trying to cast or assigning a device to a model after converting it in 8 bit will throw an error checks also if other models are casted correctly tries with str tries with a dtype tries with a device tries with a device tries with a device test if we did not break anything check this does not throw an error check this does not throw an error check this does not throw an error test whether it is possible to mix both 4bit and fp32 weights when using keep_in_fp32_modules correctly flan t5 uses dense act instead of dense relu dense teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch org t how can we release gpu memory cache 14530 27 test whether it is possible to mix both 4bit and fp32 weights when using keep_in_fp32_modules correctly flan t5 small uses t5densegatedactdense whereas t5 small uses t5densereludense we need to test both cases test with t5 small test with flan t5 small test whether it is possible to mix both 4bit and fp32 weights when using keep_in_fp32_modules correctly flan t5 small uses t5densegatedactdense whereas t5 small uses t5densereludense we need to test both cases test with t5 small there was a bug with decoders this test checks that it is fixed test with flan t5 small model_name different types of model sequence classification model causallm model seq2seq model teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch org t how can we release gpu memory cache 14530 27 a simple test to check if the last modules for some classes automodelforcausallm or sequenceclassification are kept in their native class other heads should be nn parameter teardown function needs to be called at the end of each test to free the gpu memory and cache also to avoid unexpected behaviors please see https discuss pytorch org t how can we release gpu memory cache 14530 27 the aim of this test is to verify that the mixed 4bit is compatible with pipeline from transformers since we used pipline for inference speed benchmarking we want to make sure that this feature does not break anything on pipline self _clear_cuda_cache real second forward pass this tests that the model has been loaded and can be used correctly on a multi gpu setup let s just try to load a model on 2 gpus and see if it works the model we test has 2gb of total 3gb should suffice check correct device map check that inference pass works on the model second real batch step 1 freeze all parameters freeze the model train adapters later cast the small parameters e g layernorm to fp32 for stability step 2 add adapters step 3 dummy batch step 4 check if the gradient is not none
import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeq2SeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)


def get_some_linear_layer(model):
    if model.config.model_type == "gpt2":
        return model.transformer.h[0].mlp.c_fc
    return model.transformer.h[0].mlp.dense_4h_to_h


if is_torch_available():
    import torch
    import torch.nn as nn

    class LoRALayer(nn.Module):
        

        def __init__(self, module: nn.Module, rank: int):
            super().__init__()
            self.module = module
            self.adapter = nn.Sequential(
                nn.Linear(module.in_features, rank, bias=False),
                nn.Linear(rank, module.out_features, bias=False),
            )
            small_std = (2.0 / (5 * min(module.in_features, module.out_features))) ** 0.5
            nn.init.normal_(self.adapter[0].weight, std=small_std)
            nn.init.zeros_(self.adapter[1].weight)
            self.adapter.to(module.weight.device)

        def forward(self, input, *args, **kwargs):
            return self.module(input, *args, **kwargs) + self.adapter(input)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class Base4bitTest(unittest.TestCase):
    

    
    
    model_name = "bigscience/bloom-1b7"

    
    EXPECTED_RELATIVE_DIFFERENCE = (
        2.109659552692574  
    )

    input_text = "Hello my name is"
    EXPECTED_OUTPUTS = set()
    EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I")
    EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n")
    EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University")
    MAX_NEW_TOKENS = 10

    def setUp(self):
        
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)


class Bnb4BitTest(Base4bitTest):
    def setUp(self):
        super().setUp()

        
        self.model_fp16 = AutoModelForCausalLM.from_pretrained(
            self.model_name, torch_dtype=torch.float16, device_map="auto"
        )
        self.model_4bit = AutoModelForCausalLM.from_pretrained(self.model_name, load_in_4bit=True, device_map="auto")

    def tearDown(self):
        r
        del self.model_fp16
        del self.model_4bit

        gc.collect()
        torch.cuda.empty_cache()

    def test_quantization_num_parameters(self):
        r
        num_params_4bit = self.model_4bit.num_parameters()
        num_params_fp16 = self.model_fp16.num_parameters()

        self.assertEqual(num_params_4bit, num_params_fp16)

    def test_quantization_config_json_serialization(self):
        r
        config = self.model_4bit.config

        self.assertTrue(hasattr(config, "quantization_config"))

        _ = config.to_dict()
        _ = config.to_diff_dict()

        _ = config.to_json_string()

    def test_memory_footprint(self):
        r
        from bitsandbytes.nn import Params4bit

        mem_fp16 = self.model_fp16.get_memory_footprint()
        mem_4bit = self.model_4bit.get_memory_footprint()

        self.assertAlmostEqual(mem_fp16 / mem_4bit, self.EXPECTED_RELATIVE_DIFFERENCE)
        linear = get_some_linear_layer(self.model_4bit)
        self.assertTrue(linear.weight.__class__ == Params4bit)

    def test_original_dtype(self):
        r
        self.assertTrue(hasattr(self.model_4bit.config, "_pre_quantization_dtype"))
        self.assertFalse(hasattr(self.model_fp16.config, "_pre_quantization_dtype"))
        self.assertTrue(self.model_4bit.config._pre_quantization_dtype == torch.float16)

    def test_linear_are_4bit(self):
        r
        from transformers import T5PreTrainedModel

        self.model_fp16.get_memory_footprint()
        self.model_4bit.get_memory_footprint()

        for name, module in self.model_4bit.named_modules():
            if isinstance(module, torch.nn.Linear):
                if name not in ["lm_head"] + T5PreTrainedModel._keep_in_fp32_modules:
                    
                    self.assertTrue(module.weight.dtype == torch.uint8)

    def test_rwkv_4bit(self):
        r
        model_id = "RWKV/rwkv-4-169m-pile"

        quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True)

        model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config)
        tok = AutoTokenizer.from_pretrained(model_id)

        text = "Hello my name is"
        input_ids = tok.encode(text, return_tensors="pt").to(0)

        _ = model.generate(input_ids, max_new_tokens=30)

    def test_generate_quality(self):
        r
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt")
        output_sequences = self.model_4bit.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10)

        self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS)

    def test_generate_quality_config(self):
        r
        bnb_config = BitsAndBytesConfig()
        bnb_config.load_in_4bit = True

        model_4bit_from_config = AutoModelForCausalLM.from_pretrained(
            self.model_name, quantization_config=bnb_config, device_map="auto"
        )

        encoded_input = self.tokenizer(self.input_text, return_tensors="pt")
        output_sequences = model_4bit_from_config.generate(
            input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10
        )

        self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS)

    def test_raise_on_save_pretrained(self):
        r
        with self.assertRaises(NotImplementedError), tempfile.TemporaryDirectory() as tmpdirname:
            self.model_4bit.save_pretrained(tmpdirname)

    def test_raise_if_config_and_load_in_4bit(self):
        r
        bnb_config = BitsAndBytesConfig()

        with self.assertRaises(ValueError):
            _ = AutoModelForCausalLM.from_pretrained(
                self.model_name,
                quantization_config=bnb_config,
                load_in_4bit=True,
                device_map="auto",
                bnb_4bit_quant_type="nf4",
            )

    def test_device_and_dtype_assignment(self):
        r
        with self.assertRaises(ValueError):
            
            self.model_4bit.to("cpu")

        with self.assertRaises(ValueError):
            
            self.model_4bit.to(torch.float16)

        with self.assertRaises(ValueError):
            
            self.model_4bit.to(torch.device("cuda:0"))

        with self.assertRaises(ValueError):
            
            self.model_4bit.float()

        with self.assertRaises(ValueError):
            
            self.model_4bit.half()

        
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt")

        self.model_fp16 = self.model_fp16.to(torch.float32)
        _ = self.model_fp16.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10)

        
        _ = self.model_fp16.to("cpu")

        
        _ = self.model_fp16.half()

        
        _ = self.model_fp16.float()

    def test_fp32_4bit_conversion(self):
        r
        model = AutoModelForSeq2SeqLM.from_pretrained("t5-small", load_in_4bit=True, device_map="auto")
        self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.float32)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class Bnb4BitT5Test(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.model_name = "t5-small"
        cls.dense_act_model_name = "google/flan-t5-small"  
        cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name)
        cls.input_text = "Translate in German: Hello, my dog is cute"

    def tearDown(self):
        r
        gc.collect()
        torch.cuda.empty_cache()

    def test_inference_without_keep_in_fp32(self):
        r
        from transformers import T5ForConditionalGeneration

        modules = T5ForConditionalGeneration._keep_in_fp32_modules
        T5ForConditionalGeneration._keep_in_fp32_modules = None

        
        model = T5ForConditionalGeneration.from_pretrained(self.model_name, load_in_4bit=True, device_map="auto")
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt").to(0)
        _ = model.generate(**encoded_input)

        
        model = T5ForConditionalGeneration.from_pretrained(
            self.dense_act_model_name, load_in_4bit=True, device_map="auto"
        )
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt").to(0)
        _ = model.generate(**encoded_input)
        T5ForConditionalGeneration._keep_in_fp32_modules = modules

    def test_inference_with_keep_in_fp32(self):
        r
        import bitsandbytes as bnb

        from transformers import T5ForConditionalGeneration

        
        model = T5ForConditionalGeneration.from_pretrained(self.model_name, load_in_4bit=True, device_map="auto")

        
        self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q, bnb.nn.Linear4bit))

        encoded_input = self.tokenizer(self.input_text, return_tensors="pt").to(0)
        _ = model.generate(**encoded_input)

        
        model = T5ForConditionalGeneration.from_pretrained(
            self.dense_act_model_name, load_in_4bit=True, device_map="auto"
        )
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt").to(0)
        _ = model.generate(**encoded_input)


class Classes4BitModelTest(Base4bitTest):
    def setUp(self):
        super().setUp()
        
        self.model_name = "bigscience/bloom-560m"
        self.seq_to_seq_name = "t5-small"

        

        self.base_model = AutoModel.from_pretrained(self.model_name, load_in_4bit=True, device_map="auto")
        
        self.sequence_model = AutoModelForSequenceClassification.from_pretrained(
            self.model_name, load_in_4bit=True, device_map="auto"
        )
        
        self.model_4bit = AutoModelForCausalLM.from_pretrained(self.model_name, load_in_4bit=True, device_map="auto")
        
        self.seq_to_seq_model = AutoModelForSeq2SeqLM.from_pretrained(
            self.seq_to_seq_name, load_in_4bit=True, device_map="auto"
        )

    def tearDown(self):
        r
        del self.base_model
        del self.sequence_model
        del self.model_4bit
        del self.seq_to_seq_model

        gc.collect()
        torch.cuda.empty_cache()

    def test_correct_head_class(self):
        r
        from bitsandbytes.nn import Params4bit

        self.assertTrue(self.base_model.h[-1].mlp.dense_4h_to_h.weight.__class__ == Params4bit)

        
        self.assertTrue(self.model_4bit.lm_head.weight.__class__ == torch.nn.Parameter)
        self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter)
        self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter)


class Pipeline4BitTest(Base4bitTest):
    def setUp(self):
        super().setUp()

    def tearDown(self):
        r
        del self.pipe

        gc.collect()
        torch.cuda.empty_cache()

    def test_pipeline(self):
        r
        
        self.pipe = pipeline(
            "text-generation",
            model=self.model_name,
            model_kwargs={"device_map": "auto", "load_in_4bit": True, "torch_dtype": torch.float16},
            max_new_tokens=self.MAX_NEW_TOKENS,
        )

        
        pipeline_output = self.pipe(self.input_text)
        self.assertIn(pipeline_output[0]["generated_text"], self.EXPECTED_OUTPUTS)


@require_torch_multi_gpu
class Bnb4bitTestMultiGpu(Base4bitTest):
    def setUp(self):
        super().setUp()

    def test_multi_gpu_loading(self):
        r

        model_parallel = AutoModelForCausalLM.from_pretrained(
            self.model_name, load_in_4bit=True, device_map="balanced"
        )

        
        self.assertEqual(set(model_parallel.hf_device_map.values()), {0, 1})

        
        encoded_input = self.tokenizer(self.input_text, return_tensors="pt")

        
        output_parallel = model_parallel.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10)
        self.assertIn(self.tokenizer.decode(output_parallel[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS)


class Bnb4BitTestTraining(Base4bitTest):
    def setUp(self):
        self.model_name = "facebook/opt-350m"
        super().setUp()

    def test_training(self):
        if version.parse(importlib.metadata.version("bitsandbytes")) < version.parse("0.37.0"):
            return

        
        model = AutoModelForCausalLM.from_pretrained(self.model_name, load_in_4bit=True)

        self.assertEqual(set(model.hf_device_map.values()), {torch.cuda.current_device()})

        for param in model.parameters():
            param.requires_grad = False  
            if param.ndim == 1:
                
                param.data = param.data.to(torch.float32)

        
        for _, module in model.named_modules():
            if "OPTAttention" in repr(type(module)):
                module.q_proj = LoRALayer(module.q_proj, rank=16)
                module.k_proj = LoRALayer(module.k_proj, rank=16)
                module.v_proj = LoRALayer(module.v_proj, rank=16)

        
        batch = self.tokenizer("Test batch ", return_tensors="pt").to(0)

        
        with torch.cuda.amp.autocast():
            out = model.forward(**batch)
            out.logits.norm().backward()

        for module in model.modules():
            if isinstance(module, LoRALayer):
                self.assertTrue(module.adapter[1].weight.grad is not None)
                self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0)
            elif isinstance(module, nn.Embedding):
                self.assertTrue(module.weight.grad is None)


class Bnb4BitGPT2Test(Bnb4BitTest):
    model_name = "gpt2-xl"
    EXPECTED_RELATIVE_DIFFERENCE = 3.3191854854152187