Upload 2 files

app.py

@@ -6,8 +6,8 @@ import gradio as gr
 import spaces
 import torch
 
-from awq import AutoAWQForCausalLM
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, AwqConfig
 
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
@@ -16,12 +16,19 @@ ACCESS_TOKEN = os.getenv("HF_TOKEN", "")
 
 model_id = "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4"
 
-model = AutoAWQForCausalLM.from_pretrained(
+quantization_config = AwqConfig(
+    bits=4,
+    fuse_max_seq_len=512, # Note: Update this as per your use-case
+    do_fuse=True,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
     model_id,
     torch_dtype=torch.float16,
     device_map="auto",
     trust_remote_code=True,
     low_cpu_mem_usage=True,
+    quantization_config=quantization_config,
     token=ACCESS_TOKEN)
 tokenizer = AutoTokenizer.from_pretrained(
     model_id,

requirements.txt

@@ -246,4 +246,3 @@ einops
 pytest
 gguf>=0.10.0
 autoawq
-awq