Spaces:
Sleeping
Sleeping
File size: 1,180 Bytes
aa72bf4 70bce58 aa72bf4 56e049f aa72bf4 cb562be aa72bf4 66b0d55 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
from ctransformers import AutoModelForCausalLM
from fastapi import FastAPI
from pydantic import BaseModel
from llama_cpp import Llama
llm = Llama.from_pretrained(
repo_id="TheBloke/TowerInstruct-7B-v0.1-GGUF",
filename="towerinstruct-7b-v0.1.Q5_K_M.gguf",
n_ctx = 4096,
)
#Pydantic object
class validation(BaseModel):
prompt: str
#Fast API
app = FastAPI()
def translate_zh_to_en(llm, text):
response = llm.create_chat_completion(
messages = [
{
"role": "user",
"content": f"Translate the following text from Chinese into English.\nChinese: {text}\nEnglish:"
}
],
temperature=0.2,
max_tokens=2048
)
# Assuming the response from llm.create_chat_completion is stored in a variable called response
content = response['choices'][0]['message']['content']
return content
# <|im_start|>user
# Translate the following text from Portuguese into English.
# Portuguese: Um grupo de investigadores lançou um novo modelo para tarefas relacionadas com tradução.
# English:<|im_end|>
# <|im_start|>assistant
@app.post("/translate")
async def stream(item: validation):
return translate_zh_to_en(llm, item.prompt)
|