File size: 1,581 Bytes
4900ca9
 
 
 
 
 
 
 
 
 
 
 
 
cd0f47e
4900ca9
1ae6855
4900ca9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1ae6855
4900ca9
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import sys
sys.path.insert(1, '/workspace/asr/peft/src')
# TODO set this path to the lazy-lora source code path, or you can install it from source code:
# TODO, please install lazylora for usage:
# git clone [email protected]:Xianchao-Wu/peft.git
# cd peft
# python setup.py install

from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import PeftModel, PeftConfig
import os
import torch

#import ipdb; ipdb.set_trace()
cache_dir="/workspace/asr/peft/qlora" 
# TODO set this cache_dir to the path where you stored (or, want to store) llama2-13b-hf model

lazylora_dir=os.getcwd() # the path that contains 'adapter_config.json' and 'adapter_model.bin'

config = PeftConfig.from_pretrained(lazylora_dir)

tokenizer = AutoTokenizer.from_pretrained(
    config.base_model_name_or_path, 
    cache_dir=cache_dir,
    use_auth_token=True
)

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type='nf4',
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    config.base_model_name_or_path,
    quantization_config=bnb_config,
    device_map="auto",
    cache_dir=cache_dir,
    use_auth_token=True
)
#model.print_trainable_parameters()
print(sum(p.numel() for p in model.parameters()))
# 6,671,979,520 -> half-size of 13B due to 4-bit loading

model = PeftModel.from_pretrained(model, lazylora_dir)
print('after adding lazy lora parameters:')
model.print_trainable_parameters()
# trainable params: 0 || all params: 6,922,290,688 || trainable%: 0.0