Spaces:
Sleeping
Sleeping
import gradio as gr | |
from llama_cpp import Llama | |
llm = Llama.from_pretrained( | |
repo_id="maddes8cht/smallcloudai-Refact-1_6B-fim-gguf", | |
filename="smallcloudai-Refact-1_6B-fim-Q4_K_M.gguf", | |
numa=True, | |
seed=-1, | |
n_batch=1024, | |
n_ctx=4095, | |
) | |
def respond(prompt: str): | |
print(llm(prompt, max_tokens=64)) | |
return prompt | |
prompt = "<fim_prefix>import socket\n\ndef ping_exponential_backoff():\n <fim_suffix>\n\nif __name__ == \"main\":\n ping_exponential_backoff()<fim_middle>" | |
demo = gr.Interface(fn=respond, inputs=[gr.TextArea(prompt)], outputs=[gr.TextArea()]) | |
demo.launch(server_name="0.0.0.0", server_port=7860) | |