|
--- |
|
library_name: transformers |
|
license: bigcode-openrail-m |
|
base_model: |
|
- bigcode/starcoder2-3b |
|
--- |
|
|
|
Bitsandbytes quantization of https://huggingface.co/bigcode/starcoder2-3b. See https://huggingface.co/blog/4bit-transformers-bitsandbytes for instructions. |
|
|
|
from transformers import AutoModelForCausalLM |
|
from transformers import BitsAndBytesConfig |
|
import torch |
|
|
|
nf4_config = BitsAndBytesConfig( |
|
load_in_4bit=True, |
|
bnb_4bit_quant_type="nf4", |
|
bnb_4bit_use_double_quant=True, |
|
bnb_4bit_compute_dtype=torch.bfloat16 |
|
) |
|
model = AutoModelForCausalLM.from_pretrained("bigcode/starcoder2-3b", quantization_config=nf4_config) |
|
model.push_to_hub("onekq-ai/starcoder2-3b-bnb-4bit") |
