File size: 3,673 Bytes
33999dc f8551c3 33999dc 5f8283a fde56df 5f8283a 0cb51e1 5f8283a 30392ea 5f8283a 8abd220 5f8283a 87c8290 5f8283a 0cb51e1 5f8283a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 |
---
license: llama3.1
datasets:
- liuhaotian/LLaVA-CC3M-Pretrain-595K
pipeline_tag: image-text-to-text
---
# llama-3.1-8B-vision-378
Projection module trained to add vision capabilties to Llama 3 using SigLIP, then applied to Llama-3.1-8B-Instruct. Built by [@yeswondwerr](https://x.com/yeswondwerr) and [@qtnx_](https://x.com/qtnx_).
## Usage
```python
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from PIL import Image
import requests
from io import BytesIO
url = "https://huggingface.co/qresearch/llama-3-vision-alpha-hf/resolve/main/assets/demo-2.jpg"
response = requests.get(url)
image = Image.open(BytesIO(response.content))
model = AutoModelForCausalLM.from_pretrained(
"qresearch/llama-3.1-8B-vision-378",
trust_remote_code=True,
torch_dtype=torch.float16,
).to("cuda")
tokenizer = AutoTokenizer.from_pretrained("qresearch/llama-3.1-8B-vision-378", use_fast=True,)
print(
model.answer_question(
image, "Briefly describe the image", tokenizer, max_new_tokens=128, do_sample=True, temperature=0.3
),
)
```
## 4-bit quantization
```python
import torch
from PIL import Image
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers import BitsAndBytesConfig
import requests
from io import BytesIO
url = "https://huggingface.co/qresearch/llama-3-vision-alpha-hf/resolve/main/assets/demo-2.jpg"
response = requests.get(url)
image = Image.open(BytesIO(response.content))
bnb_cfg = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.float16,
llm_int8_skip_modules=["mm_projector", "vision_model"],
)
model = AutoModelForCausalLM.from_pretrained(
"qresearch/llama-3.1-8B-vision-378",
trust_remote_code=True,
torch_dtype=torch.float16,
quantization_config=bnb_cfg,
)
tokenizer = AutoTokenizer.from_pretrained(
"qresearch/llama-3.1-8B-vision-378",
use_fast=True,
)
print(
model.answer_question(
image, "Briefly describe the image", tokenizer, max_new_tokens=128, do_sample=True, temperature=0.3
),
)
```
```
.x+=:.
z` ^% .uef^"
.u . . <k .u . :d88E
.u@u .d88B :@8c .u .@8Ned8" .u u .d88B :@8c . `888E
.zWF8888bx ="8888f8888r ud8888. .@^%8888" ud8888. us888u. ="8888f8888r .udR88N 888E .z8k
.888 9888 4888>'88" :888'8888. x88: `)8b. :888'8888. .@88 "8888" 4888>'88" <888'888k 888E~?888L
I888 9888 4888> ' d888 '88%" 8888N=*8888 d888 '88%" 9888 9888 4888> ' 9888 'Y" 888E 888E
I888 9888 4888> 8888.+" %8" R88 8888.+" 9888 9888 4888> 9888 888E 888E
I888 9888 .d888L .+ 8888L @8Wou 9% 8888L 9888 9888 .d888L .+ 9888 888E 888E
`888Nx?888 ^"8888*" '8888c. .+ .888888P` '8888c. .+ 9888 9888 ^"8888*" ?8888u../ 888E 888E
"88" '888 "Y" "88888% ` ^"F "88888% "888*""888" "Y" "8888P' m888N= 888>
88E "YP' "YP' ^Y" ^Y' "P' `Y" 888
98> J88"
'8 @%
` :"
``` |