quuestion_paper

question_paper.py

@@ -1,47 +1,34 @@
-from pydantic import BaseModel
+import time
+import copy
+import asyncio
+import requests
 
-from .ConfigEnv import config
-from fastapi.middleware.cors import CORSMiddleware
+from fastapi import FastAPI, Request
+from llama_cpp import Llama
+from sse_starlette import EventSourceResponse
+# Load the model
+print("Loading model...")
+llm = Llama(model_path="./llama-2-13b-chat.ggmlv3.q4_1.bin") # change based on the location of models
+print("Model loaded!")
 
-from langchain.llms import Clarifai
-from langchain.chains import LLMChain
-from langchain.prompts import PromptTemplate
-from TextGen import app
+app = FastAPI()
 
-class Generate(BaseModel):
-    text:str
-
-def generate_text(prompt: str):
-    if prompt == "":
-        return {"detail": "Please provide a prompt."}
-    else:
-        prompt = PromptTemplate(template=prompt, input_variables=['Prompt'])
-        llm = Clarifai(
-            pat = config.CLARIFAI_PAT,
-            user_id = config.USER_ID,
-            app_id = config.APP_ID, 
-            model_id = config.MODEL_ID,
-            model_version_id=config.MODEL_VERSION_ID,
-        )
-        llmchain = LLMChain(
-            prompt=prompt,
-            llm=llm
-        )
-        llm_response = llmchain.run({"Prompt": prompt})
-        return Generate(text=llm_response)
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-@app.get("/", tags=["Home"])
-def api_home():
-    return {'detail': 'Welcome to FastAPI TextGen Tutorial!'}
-
-@app.post("/api/generate", summary="Generate text from prompt", tags=["Generate"], response_model=Generate)
-def inference(input_prompt: str):
-    return generate_text(prompt=input_prompt)
+@app.get("/llama")
+async def llama(request: Request, question:str):
+    stream = llm(
+        f"""{question}""",
+        max_tokens=100,
+        stop=["\n", " Q:"],
+        stream=True,
+    )
+    async def async_generator():
+        for item in stream:
+            yield item
+    async def server_sent_events():
+        async for item in async_generator():
+            if await request.is_disconnected():
+                break
+            result = copy.deepcopy(item)
+            text = result["choices"][0]["text"]
+            yield {"data": text}
+    return EventSourceResponse(server_sent_events())