MediaTek-Research
/

Breeze-7B-FC-v1_0

Model card Files Files and versions Community

YC-Chen commited on 3 days ago

Commit

8c55329

•

1 Parent(s): 4915dcb

Update README.md

Files changed (1) hide show

README.md +2 -1

README.md CHANGED Viewed

@@ -88,7 +88,8 @@ from vllm import LLM, SamplingParams
 llm = LLM(
     model='MediaTek-Research/Breeze-7B-FC-v1_0',
     tensor_parallel_size=num_gpu, # number of gpus
-    gpu_memory_utilization=0.7
 )
 turn_end_token_id = 61876 # <|im_end|>

 llm = LLM(
     model='MediaTek-Research/Breeze-7B-FC-v1_0',
     tensor_parallel_size=num_gpu, # number of gpus
+    gpu_memory_utilization=0.7,
+    dtype='half'
 )
 turn_end_token_id = 61876 # <|im_end|>